00000100:00000010:3.0F:1790701405.169073:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908daf100. 02000000:00000010:3.0:1790701405.169125:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d656a00. 00000100:00000010:3.0:1790701405.169204:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908dae300. 02000000:00000010:3.0:1790701405.169222:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d656300. 02000000:00000010:3.0:1790701405.169266:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a176600. 00000400:00000200:3.0:1790701405.169279:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701405.169284:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701405.169293:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701405.169303:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323521920, portal 10 00000400:00000010:3.0:1790701405.169311:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35dd0. 00000100:00000200:3.0:1790701405.169316:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323521920, offset 0 00000400:00000200:3.0:1790701405.169327:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701405.169341:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701405.169352:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701405.169358:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701405.169362:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701405.169366:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701405.169370:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701405.169373:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [86:86:256:4294967295] 192.168.206.116@tcp LPNI seq info [86:86:8:4294967295] 00000400:00000200:3.0:1790701405.169378:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701405.169389:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701405.169396:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701405.169401:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d656d00. 00000800:00000200:3.0:1790701405.169408:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701405.169417:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701405.169422:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d656d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701405.169444:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701405.169449:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c93b3e3ae0. 00000100:00000010:3.0:1790701405.169466:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c93b3e3f60. 02000000:00000010:3.0:1790701405.169488:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a177c00. 00000400:00000200:3.0:1790701405.169493:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701405.169495:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9f20. 00000400:00000010:3.0:1790701405.169505:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d353b8. 00000100:00000200:3.0:1790701405.169508:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323521792, portal 25 00000400:00000010:3.0:1790701405.169512:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35110. 00000100:00000200:3.0:1790701405.169514:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323521792, offset 0 00000400:00000200:3.0:1790701405.169519:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701405.169523:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701405.169526:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701405.169542:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701405.169544:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701405.169554:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701405.169556:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701405.169559:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [87:87:255:4294967295] 192.168.206.116@tcp LPNI seq info [87:87:7:4294967295] 00000400:00000200:3.0:1790701405.169563:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701405.169569:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701405.169574:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701405.169578:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d656000. 00000800:00000200:3.0:1790701405.169582:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701405.169586:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701405.169588:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d656000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701405.169592:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0F:1790701405.170885:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701405.170906:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d656d00. 00000400:00000200:1.0:1790701405.170915:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701405.170925:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701405.170931:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35dd0 00000400:00000010:1.0:1790701405.170934:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35dd0. 00000100:00000200:1.0:1790701405.170945:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908dae300 x1877686323521920/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701405.171007:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701405.171009:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d656000. 00000400:00000200:1.0:1790701405.171012:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701405.171016:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701405.171019:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35110 00000400:00000010:1.0:1790701405.171021:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35110. 00000100:00000200:1.0:1790701405.171026:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908daf100 x1877686323521792/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701421 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0F:1790701405.176501:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701405.176543:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701405.176554:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701405.176560:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701405.176566:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701405.176570:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502900 00000400:00000200:0.0:1790701405.176577:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701405.176584:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x5f9 [1] + 224 00000400:00000200:0.0:1790701405.176598:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701405.176603:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701405.176620:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908daf100 x1877686323521792/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701421 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701405.176661:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701405.176665:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701405.176669:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701405.176672:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701405.177425:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701405.177431:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502980 00000400:00000200:0.0:1790701405.177436:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701405.177442:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x5f1 [1] + 224 00000400:00000200:0.0:1790701405.177460:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701405.177465:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701405.177474:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908dae300 x1877686323521920/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701421 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701405.177881:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9f20. 00000400:00000200:3.0:1790701405.177887:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d353b8 00000400:00000010:3.0:1790701405.177889:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d353b8. 00000100:00000200:3.0:1790701405.177898:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908daf100 x1877686323521792/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701421 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701405.177907:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908daf100 x1877686323521792/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701421 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701405.177932:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:3.0:1790701405.177934:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198 00000400:00000010:3.0:1790701405.177936:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701405.177939:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908dae300 x1877686323521920/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701421 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701405.177960:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908dae300 x1877686323521920/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701421 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701405.177984:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a177c00. 02000000:00000010:3.0:1790701405.177992:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d656a00. 00000100:00000010:3.0:1790701405.177997:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908daf100. 02000000:00000010:3.0:1790701405.178001:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a176600. 02000000:00000010:3.0:1790701405.178003:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d656300. 00000100:00000010:3.0:1790701405.178005:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908dae300. 00000100:00000010:1.0:1790701406.124456:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c93b3e3f60. 00000100:00000010:0.0:1790701406.126914:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c93b3e3ae0. 00000100:00000010:0.0:1790701410.284954:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a23480. 02000000:00000010:0.0:1790701410.284984:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3700. 00000100:00000010:0.0:1790701410.285034:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a21880. 02000000:00000010:0.0:1790701410.285040:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3a00. 02000000:00000010:0.0:1790701410.285084:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a042800. 00000400:00000200:0.0:1790701410.285114:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701410.285119:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2898. 00000400:00000010:0.0:1790701410.285127:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a4c8. 00000100:00000200:0.0:1790701410.285137:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323522176, portal 10 00000400:00000010:0.0:1790701410.285146:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701410.285150:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323522176, offset 0 00000400:00000200:0.0:1790701410.285207:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701410.285217:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.285226:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701410.285232:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701410.285237:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701410.285242:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701410.285245:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701410.285248:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [88:88:256:4294967295] 192.168.206.116@tcp LPNI seq info [88:88:8:4294967295] 00000400:00000200:0.0:1790701410.285253:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.285264:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701410.285271:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701410.285276:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bdb3900. 00000800:00000200:0.0:1790701410.285283:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701410.285292:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701410.285298:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bdb3900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701410.285316:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:0.0:1790701410.285323:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92249e660. 00000100:00000010:0.0:1790701410.285336:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92249e240. 02000000:00000010:0.0:1790701410.285363:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a042400. 00000400:00000200:0.0:1790701410.285368:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701410.285371:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2f20. 00000400:00000010:0.0:1790701410.285374:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a880. 00000100:00000200:0.0:1790701410.285377:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323522048, portal 25 00000400:00000010:0.0:1790701410.285380:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a660. 00000100:00000200:0.0:1790701410.285382:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323522048, offset 0 00000400:00000200:0.0:1790701410.285393:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701410.285396:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.285400:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701410.285402:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701410.285405:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701410.285407:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701410.285409:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701410.285412:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [89:89:255:4294967295] 192.168.206.116@tcp LPNI seq info [89:89:7:4294967295] 00000400:00000200:0.0:1790701410.285416:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.285422:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701410.285426:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701410.285430:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bdb3d00. 00000800:00000200:0.0:1790701410.285434:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701410.285438:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701410.285440:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bdb3d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701410.285445:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701410.285505:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701410.285509:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bdb3900. 00000400:00000200:0.0:1790701410.285517:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701410.285522:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701410.285528:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a770 00000400:00000010:0.0:1790701410.285531:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701410.285541:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a21880 x1877686323522176/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701410.285635:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701410.285638:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bdb3d00. 00000400:00000200:0.0:1790701410.285641:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701410.285644:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701410.285647:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a660 00000400:00000010:0.0:1790701410.285649:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a660. 00000100:00000200:0.0:1790701410.285664:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a23480 x1877686323522048/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701410.290091:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.290106:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701410.290117:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.290123:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.290130:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.290135:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502a00 00000400:00000200:0.0:1790701410.290143:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.290151:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x611 [1] + 224 00000400:00000200:0.0:1790701410.290821:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701410.290827:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701410.290846:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a23480 x1877686323522048/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701426 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701410.291046:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2f20. 00000400:00000200:1.0:1790701410.291053:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a880 00000400:00000010:1.0:1790701410.291055:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a880. 00000100:00000200:1.0:1790701410.291063:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a23480 x1877686323522048/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701426 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701410.291072:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a23480 x1877686323522048/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701426 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701410.291104:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a042400. 02000000:00000010:1.0:1790701410.291112:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3700. 00000100:00000010:1.0:1790701410.291116:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a23480. 00000400:00000200:0.0:1790701410.293361:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.293374:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701410.293383:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.293389:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.293396:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.293401:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502a80 00000400:00000200:0.0:1790701410.293407:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701410.293414:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x601 [1] + 224 00000400:00000200:0.0:1790701410.293432:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701410.293437:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701410.293453:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a21880 x1877686323522176/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701426 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701410.293833:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2898. 00000400:00000200:1.0:1790701410.293839:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a4c8 00000400:00000010:1.0:1790701410.293841:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a4c8. 00000100:00000200:1.0:1790701410.293849:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a21880 x1877686323522176/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701426 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701410.293958:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a21880 x1877686323522176/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701426 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701410.293998:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a042800. 02000000:00000010:1.0:1790701410.294004:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3a00. 00000100:00000010:1.0:1790701410.294009:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a21880. 00000100:00000010:1.0:1790701411.244974:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92249e240. 00000100:00000010:0.0:1790701411.246083:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92249e660. 00000001:02000400:0.0:1790701413.614391:0:128329:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 550: DD-off: keep route up when gw reachable via another local NI ========================================================== 13:03:32 (1790701412) 00000001:02000400:2.0F:1790701415.112098:0:128466:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity-lnet test_550 Need at least 2 local interfaces for multi-rail 00000100:00000010:3.0:1790701415.406857:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908dae300. 02000000:00000010:3.0:1790701415.406877:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d656000. 00000100:00000010:3.0:1790701415.406922:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908daf100. 02000000:00000010:3.0:1790701415.406927:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d656d00. 02000000:00000010:3.0:1790701415.406960:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a176600. 00000400:00000200:3.0:1790701415.406971:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701415.406975:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701415.406982:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701415.406989:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323522304, portal 25 00000400:00000010:3.0:1790701415.406994:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d353b8. 00000100:00000200:3.0:1790701415.406998:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323522304, offset 0 00000400:00000200:3.0:1790701415.407007:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701415.407015:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701415.407022:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701415.407027:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701415.407031:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701415.407035:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701415.407037:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701415.407041:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [90:90:256:4294967295] 192.168.206.116@tcp LPNI seq info [90:90:8:4294967295] 00000400:00000200:3.0:1790701415.407045:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701415.407054:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701415.407068:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701415.407073:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d656a00. 00000800:00000200:3.0:1790701415.407079:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701415.407087:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701415.407091:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d656a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701415.407111:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701415.407116:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c93b3e33c0. 00000100:00000010:3.0:1790701415.407127:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c93b3e3c00. 02000000:00000010:3.0:1790701415.407145:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a177000. 00000400:00000200:3.0:1790701415.407149:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701415.407176:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9f20. 00000400:00000010:3.0:1790701415.407181:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35908. 00000100:00000200:3.0:1790701415.407184:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323522432, portal 10 00000400:00000010:3.0:1790701415.407190:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35c38. 00000100:00000200:3.0:1790701415.407192:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323522432, offset 0 00000400:00000200:3.0:1790701415.407197:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701415.407201:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701415.407204:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701415.407207:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701415.407209:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701415.407212:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701415.407214:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701415.407217:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [91:91:255:4294967295] 192.168.206.116@tcp LPNI seq info [91:91:7:4294967295] 00000400:00000200:3.0:1790701415.407221:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701415.407227:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701415.407231:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701415.407235:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f300. 00000800:00000200:3.0:1790701415.407240:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701415.407244:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701415.407247:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701415.407260:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701415.407523:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701415.407532:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d656a00. 00000400:00000200:0.0:1790701415.407538:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701415.407543:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701415.407548:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d353b8 00000400:00000010:0.0:1790701415.407551:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d353b8. 00000100:00000200:0.0:1790701415.407559:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908dae300 x1877686323522304/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701415.408517:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701415.408522:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f300. 00000400:00000200:0.0:1790701415.408526:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701415.408531:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701415.408536:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35c38 00000400:00000010:0.0:1790701415.408538:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35c38. 00000100:00000200:0.0:1790701415.408545:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908daf100 x1877686323522432/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701415.413339:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701415.413349:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701415.413357:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701415.413362:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701415.413368:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701415.413372:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502b80 00000400:00000200:0.0:1790701415.413377:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701415.413383:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x629 [1] + 224 00000400:00000200:0.0:1790701415.413395:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701415.413399:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701415.413413:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908daf100 x1877686323522432/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701431 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701415.413750:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9f20. 00000400:00000200:3.0:1790701415.413755:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35908 00000400:00000010:3.0:1790701415.413758:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35908. 00000100:00000200:3.0:1790701415.413765:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908daf100 x1877686323522432/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701431 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701415.413776:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908daf100 x1877686323522432/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701431 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701415.413805:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a177000. 02000000:00000010:3.0:1790701415.413810:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d656d00. 00000100:00000010:3.0:1790701415.413815:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908daf100. 00000400:00000200:0.0:1790701415.422031:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701415.422041:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701415.422049:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701415.422062:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701415.422067:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701415.422071:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502b00 00000400:00000200:0.0:1790701415.422077:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701415.422083:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x621 [1] + 224 00000400:00000200:0.0:1790701415.422098:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701415.422103:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701415.422115:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908dae300 x1877686323522304/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701431 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701415.422252:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:3.0:1790701415.422257:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198 00000400:00000010:3.0:1790701415.422259:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701415.422266:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908dae300 x1877686323522304/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701431 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701415.422276:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908dae300 x1877686323522304/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701431 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701415.422300:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a176600. 02000000:00000010:3.0:1790701415.422306:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d656000. 00000100:00000010:3.0:1790701415.422309:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908dae300. 00000100:00000010:0.0:1790701416.364218:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c93b3e33c0. 00000100:00000010:1.0:1790701416.364347:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c93b3e3c00. 00000001:02000400:0.0:1790701416.623346:0:128619:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 600: lnet_legacy2yaml: basic ip2nets conversion ========================================================== 13:03:35 (1790701415) 00000100:00000010:2.0:1790701420.525395:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94700. 02000000:00000010:2.0:1790701420.525423:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1a00. 00000100:00000010:2.0:1790701420.525463:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95880. 02000000:00000010:2.0:1790701420.525467:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1e00. 02000000:00000010:2.0:1790701420.525500:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c9350cb400. 00000400:00000200:2.0:1790701420.525510:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701420.525514:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823688. 00000400:00000010:2.0:1790701420.525522:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeedd0. 00000100:00000200:2.0:1790701420.525528:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323522688, portal 10 00000400:00000010:2.0:1790701420.525537:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee110. 00000100:00000200:2.0:1790701420.525540:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323522688, offset 0 00000400:00000200:2.0:1790701420.525549:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701420.525557:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701420.525564:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701420.525569:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701420.525573:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701420.525577:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701420.525579:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701420.525583:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [92:92:256:4294967295] 192.168.206.116@tcp LPNI seq info [92:92:8:4294967295] 00000400:00000200:2.0:1790701420.525587:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701420.525597:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701420.525603:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701420.525607:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1800. 00000800:00000200:2.0:1790701420.525613:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701420.525620:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701420.525624:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701420.525642:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:2.0:1790701420.525648:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305c840. 00000100:00000010:2.0:1790701420.525660:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305ccc0. 02000000:00000010:2.0:1790701420.525682:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c9350cbe00. 00000400:00000200:2.0:1790701420.525710:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701420.525713:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823898. 00000400:00000010:2.0:1790701420.525717:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeeaa0. 00000100:00000200:2.0:1790701420.525720:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323522560, portal 25 00000400:00000010:2.0:1790701420.525724:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeea18. 00000100:00000200:2.0:1790701420.525726:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323522560, offset 0 00000400:00000200:2.0:1790701420.525730:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701420.525734:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701420.525737:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701420.525740:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701420.525743:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701420.525745:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701420.525747:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701420.525750:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [93:93:255:4294967295] 192.168.206.116@tcp LPNI seq info [93:93:7:4294967295] 00000400:00000200:2.0:1790701420.525755:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701420.525760:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701420.525765:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701420.525769:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1100. 00000800:00000200:2.0:1790701420.525772:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701420.525777:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701420.525780:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701420.525784:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701420.527286:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701420.527295:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1800. 00000400:00000200:1.0:1790701420.527304:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701420.527312:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701420.527319:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee110 00000400:00000010:1.0:1790701420.527322:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee110. 00000100:00000200:1.0:1790701420.527334:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b95880 x1877686323522688/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701420.527414:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701420.527418:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1100. 00000400:00000200:1.0:1790701420.527421:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701420.527425:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701420.527428:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeea18 00000400:00000010:1.0:1790701420.527430:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeea18. 00000100:00000200:1.0:1790701420.527435:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b94700 x1877686323522560/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701420.538315:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701420.538325:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701420.538333:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701420.538338:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701420.538343:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701420.538347:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502c00 00000400:00000200:0.0:1790701420.538353:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701420.538359:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x639 [1] + 224 00000400:00000200:0.0:1790701420.538369:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701420.538374:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701420.538388:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b94700 x1877686323522560/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701436 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701420.538426:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701420.538430:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701420.538433:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701420.538435:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701420.538438:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701420.538440:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502c80 00000400:00000200:0.0:1790701420.538450:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701420.538455:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x631 [1] + 224 00000400:00000200:0.0:1790701420.538465:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701420.538468:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701420.538474:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b95880 x1877686323522688/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701436 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701420.538785:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823898. 00000400:00000200:3.0:1790701420.538790:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeeaa0 00000400:00000010:3.0:1790701420.538793:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeeaa0. 00000100:00000200:3.0:1790701420.538800:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b94700 x1877686323522560/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701436 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701420.538809:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b94700 x1877686323522560/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701436 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701420.538831:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823688. 00000400:00000200:3.0:1790701420.538834:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeedd0 00000400:00000010:3.0:1790701420.538844:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeedd0. 00000100:00000200:3.0:1790701420.538850:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b95880 x1877686323522688/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701436 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701420.538859:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b95880 x1877686323522688/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701436 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701420.538882:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c9350cbe00. 02000000:00000010:3.0:1790701420.538888:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1a00. 00000100:00000010:3.0:1790701420.538892:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94700. 02000000:00000010:3.0:1790701420.538895:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c9350cb400. 02000000:00000010:3.0:1790701420.538897:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1e00. 00000100:00000010:3.0:1790701420.538899:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95880. 00000100:00000010:1.0:1790701421.486115:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305ccc0. 00000100:00000010:0.0:1790701421.486226:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305c840. 00000001:02000400:0.0:1790701424.014958:0:129192:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 601: lnet_legacy2yaml: basic routes conversion ========================================================== 13:03:42 (1790701422) 00000100:00000010:0.0:1790701425.644231:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a21180. 02000000:00000010:0.0:1790701425.644250:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3500. 00000100:00000010:0.0:1790701425.644281:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a20700. 02000000:00000010:0.0:1790701425.644285:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3b00. 02000000:00000010:0.0:1790701425.644317:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908444c00. 00000400:00000200:0.0:1790701425.644326:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701425.644330:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2bb0. 00000400:00000010:0.0:1790701425.644337:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a660. 00000100:00000200:0.0:1790701425.644343:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323522816, portal 25 00000400:00000010:0.0:1790701425.644348:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701425.644350:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323522816, offset 0 00000400:00000200:0.0:1790701425.644358:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701425.644366:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701425.644373:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701425.644379:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701425.644383:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701425.644387:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701425.644389:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701425.644393:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [94:94:256:4294967295] 192.168.206.116@tcp LPNI seq info [94:94:8:4294967295] 00000400:00000200:0.0:1790701425.644397:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701425.644405:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701425.644425:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701425.644429:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bdb3c00. 00000800:00000200:0.0:1790701425.644436:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701425.644443:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701425.644447:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bdb3c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701425.644459:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:0.0:1790701425.644464:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92249eae0. 00000100:00000010:0.0:1790701425.644475:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92249ee40. 02000000:00000010:0.0:1790701425.644494:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908445a00. 00000400:00000200:0.0:1790701425.644498:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701425.644500:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2478. 00000400:00000010:0.0:1790701425.644503:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a7f8. 00000100:00000200:0.0:1790701425.644506:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323522944, portal 10 00000400:00000010:0.0:1790701425.644512:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45acc0. 00000100:00000200:0.0:1790701425.644514:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323522944, offset 0 00000400:00000200:0.0:1790701425.644518:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701425.644521:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701425.644524:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701425.644527:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701425.644529:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701425.644531:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701425.644533:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701425.644536:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [95:95:255:4294967295] 192.168.206.116@tcp LPNI seq info [95:95:7:4294967295] 00000400:00000200:0.0:1790701425.644540:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701425.644545:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701425.644550:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701425.644552:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bdb3100. 00000800:00000200:0.0:1790701425.644556:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701425.644559:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701425.644562:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bdb3100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701425.644573:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701425.644616:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701425.644619:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bdb3c00. 00000400:00000200:0.0:1790701425.644624:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701425.644629:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701425.644634:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a770 00000400:00000010:0.0:1790701425.644637:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701425.644645:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a21180 x1877686323522816/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701425.644708:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701425.644711:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bdb3100. 00000400:00000200:0.0:1790701425.644714:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701425.644717:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701425.644720:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45acc0 00000400:00000010:0.0:1790701425.644722:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45acc0. 00000100:00000200:0.0:1790701425.644726:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a20700 x1877686323522944/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701425.652358:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701425.652369:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701425.652377:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701425.652382:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701425.652388:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701425.652403:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502d80 00000400:00000200:1.0:1790701425.652409:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701425.652416:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x651 [1] + 224 00000400:00000200:1.0:1790701425.652432:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701425.652437:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701425.652450:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a20700 x1877686323522944/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701441 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701425.652519:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2478. 00000400:00000200:0.0:1790701425.652525:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a7f8 00000400:00000010:0.0:1790701425.652527:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a7f8. 00000100:00000200:0.0:1790701425.652534:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a20700 x1877686323522944/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701441 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701425.652543:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a20700 x1877686323522944/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701441 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701425.652575:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908445a00. 02000000:00000010:0.0:1790701425.652580:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3b00. 00000100:00000010:0.0:1790701425.652585:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a20700. 00000400:00000200:0.0:1790701425.662859:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701425.663217:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701425.663227:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701425.663233:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701425.663246:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701425.663249:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502d00 00000400:00000200:0.0:1790701425.663255:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701425.663261:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x641 [1] + 224 00000400:00000200:0.0:1790701425.663286:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701425.663291:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701425.663304:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a21180 x1877686323522816/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701441 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701425.663344:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2bb0. 00000400:00000200:0.0:1790701425.663349:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a660 00000400:00000010:0.0:1790701425.663351:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a660. 00000100:00000200:0.0:1790701425.663356:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a21180 x1877686323522816/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701441 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701425.663364:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a21180 x1877686323522816/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701441 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701425.663390:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908444c00. 02000000:00000010:0.0:1790701425.663396:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3500. 00000100:00000010:0.0:1790701425.663399:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a21180. 00000100:00000010:1.0:1790701426.606492:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92249ee40. 00000100:00000010:0.0:1790701426.607241:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92249eae0. 00000001:02000400:0.0:1790701430.756039:0:129764:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 602: lnet_legacy2yaml: routes with priority ========================================================== 13:03:49 (1790701429) 00000100:00000010:2.0:1790701430.766742:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95500. 02000000:00000010:2.0:1790701430.766841:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1a00. 00000100:00000010:2.0:1790701430.766873:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94380. 02000000:00000010:2.0:1790701430.766878:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1100. 02000000:00000010:2.0:1790701430.766911:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c93bf86200. 00000400:00000200:2.0:1790701430.766922:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701430.766926:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823e18. 00000400:00000010:2.0:1790701430.766933:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee7f8. 00000100:00000200:2.0:1790701430.766940:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323523072, portal 25 00000400:00000010:2.0:1790701430.766952:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeecc0. 00000100:00000200:2.0:1790701430.766957:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323523072, offset 0 00000400:00000200:2.0:1790701430.766967:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701430.766977:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701430.770292:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701430.770298:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701430.770302:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701430.770306:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701430.770309:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701430.770313:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [96:96:256:4294967295] 192.168.206.116@tcp LPNI seq info [96:96:8:4294967295] 00000400:00000200:2.0:1790701430.770319:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701430.770330:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701430.770336:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701430.770343:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1e00. 00000800:00000200:2.0:1790701430.770350:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701430.770357:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701430.770362:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701430.770377:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:2.0:1790701430.770384:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305cea0. 00000100:00000010:2.0:1790701430.770399:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305c7e0. 02000000:00000010:2.0:1790701430.770435:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c93bf86a00. 00000400:00000200:2.0:1790701430.770440:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701430.770443:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823ec8. 00000400:00000010:2.0:1790701430.770449:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee880. 00000100:00000200:2.0:1790701430.770456:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323523200, portal 10 00000400:00000010:2.0:1790701430.770463:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee660. 00000100:00000200:2.0:1790701430.770466:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323523200, offset 0 00000400:00000200:2.0:1790701430.770471:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701430.770477:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701430.770480:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701430.770483:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701430.770492:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701430.770495:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701430.770497:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701430.770500:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [97:97:255:4294967295] 192.168.206.116@tcp LPNI seq info [97:97:7:4294967295] 00000400:00000200:2.0:1790701430.770504:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701430.770510:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701430.770514:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701430.770517:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1800. 00000800:00000200:2.0:1790701430.770521:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701430.770525:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701430.770527:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701430.770533:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701430.770725:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701430.770730:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1e00. 00000400:00000200:1.0:1790701430.770734:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701430.770739:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701430.770745:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeecc0 00000400:00000010:1.0:1790701430.770747:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeecc0. 00000100:00000200:1.0:1790701430.770756:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b95500 x1877686323523072/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701430.770828:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701430.770831:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1800. 00000400:00000200:1.0:1790701430.770834:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701430.770837:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701430.770840:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee660 00000400:00000010:1.0:1790701430.770842:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee660. 00000100:00000200:1.0:1790701430.770846:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b94380 x1877686323523200/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701430.780781:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701430.780792:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701430.780801:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701430.780807:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701430.780812:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701430.780816:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502e80 00000400:00000200:1.0:1790701430.780821:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701430.780835:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x669 [1] + 224 00000400:00000200:1.0:1790701430.780849:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701430.780854:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701430.780868:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b94380 x1877686323523200/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701446 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701430.781328:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701430.781333:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701430.781337:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701430.781340:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701430.781343:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701430.781346:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502e00 00000400:00000200:1.0:1790701430.781350:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701430.781355:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x661 [1] + 224 00000400:00000200:1.0:1790701430.781364:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701430.781367:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701430.781374:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b95500 x1877686323523072/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701446 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701430.782751:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823ec8. 00000400:00000200:2.0:1790701430.782760:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee880 00000400:00000010:2.0:1790701430.782762:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee880. 00000100:00000200:2.0:1790701430.782772:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b94380 x1877686323523200/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701446 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701430.782783:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b94380 x1877686323523200/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701446 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701430.782811:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823e18. 00000400:00000200:2.0:1790701430.782814:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee7f8 00000400:00000010:2.0:1790701430.782816:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee7f8. 00000100:00000200:2.0:1790701430.782820:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b95500 x1877686323523072/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701446 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701430.782828:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b95500 x1877686323523072/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701446 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701430.782842:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c93bf86a00. 02000000:00000010:2.0:1790701430.782848:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1100. 00000100:00000010:2.0:1790701430.782852:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94380. 02000000:00000010:2.0:1790701430.782862:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c93bf86200. 02000000:00000010:2.0:1790701430.782865:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1a00. 00000100:00000010:2.0:1790701430.782867:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95500. 00000100:00000010:0.0:1790701431.724205:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305cea0. 00000100:00000010:1.0:1790701431.724211:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305c7e0. 00000100:00000010:2.0:1790701435.885258:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95500. 02000000:00000010:2.0:1790701435.885278:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1c00. 00000100:00000010:2.0:1790701435.885316:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94380. 02000000:00000010:2.0:1790701435.885321:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1400. 02000000:00000010:2.0:1790701435.885350:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a176e00. 00000400:00000200:2.0:1790701435.885360:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701435.885363:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823e18. 00000400:00000010:2.0:1790701435.885370:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee7f8. 00000100:00000200:2.0:1790701435.885376:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323523456, portal 10 00000400:00000010:2.0:1790701435.885384:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee880. 00000100:00000200:2.0:1790701435.885387:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323523456, offset 0 00000400:00000200:2.0:1790701435.885398:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701435.885407:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701435.885415:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701435.885420:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701435.885424:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701435.885428:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701435.885430:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701435.885433:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [98:98:256:4294967295] 192.168.206.116@tcp LPNI seq info [98:98:8:4294967295] 00000400:00000200:2.0:1790701435.885437:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701435.885448:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701435.885453:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701435.885458:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1d00. 00000800:00000200:2.0:1790701435.885464:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701435.885471:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701435.885475:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701435.885506:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:2.0:1790701435.885513:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305c540. 00000100:00000010:2.0:1790701435.885532:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305cba0. 02000000:00000010:2.0:1790701435.885556:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a176c00. 00000400:00000200:2.0:1790701435.885562:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701435.885565:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823ec8. 00000400:00000010:2.0:1790701435.885570:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee088. 00000100:00000200:2.0:1790701435.885573:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323523328, portal 25 00000400:00000010:2.0:1790701435.885577:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee4c8. 00000100:00000200:2.0:1790701435.885580:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323523328, offset 0 00000400:00000200:2.0:1790701435.885585:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701435.885589:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701435.885593:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701435.885596:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701435.885598:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701435.885600:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701435.885602:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701435.885605:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [99:99:255:4294967295] 192.168.206.116@tcp LPNI seq info [99:99:7:4294967295] 00000400:00000200:2.0:1790701435.885609:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701435.885616:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701435.885621:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701435.885625:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1800. 00000800:00000200:2.0:1790701435.885629:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701435.885633:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701435.885636:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701435.885641:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701435.885801:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701435.885807:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1d00. 00000400:00000200:1.0:1790701435.885814:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701435.885820:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701435.885825:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee880 00000400:00000010:1.0:1790701435.885827:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee880. 00000100:00000200:1.0:1790701435.885836:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b94380 x1877686323523456/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701435.885910:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701435.885913:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1800. 00000400:00000200:1.0:1790701435.885916:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701435.885920:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701435.885929:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee4c8 00000400:00000010:1.0:1790701435.885930:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee4c8. 00000100:00000200:1.0:1790701435.885936:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b95500 x1877686323523328/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701435.890714:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701435.890725:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701435.890733:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701435.890738:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701435.890744:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701435.890748:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502f80 00000400:00000200:1.0:1790701435.890753:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701435.890759:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x671 [1] + 224 00000400:00000200:1.0:1790701435.890773:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701435.890778:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701435.890791:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b94380 x1877686323523456/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701451 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701435.890885:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823e18. 00000400:00000200:2.0:1790701435.890890:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee7f8 00000400:00000010:2.0:1790701435.890893:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee7f8. 00000100:00000200:2.0:1790701435.890900:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b94380 x1877686323523456/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701451 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701435.890910:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b94380 x1877686323523456/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701451 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701435.890955:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a176e00. 02000000:00000010:2.0:1790701435.890960:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1400. 00000100:00000010:2.0:1790701435.890965:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94380. 00000400:00000200:1.0:1790701435.896363:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701435.896373:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701435.896382:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701435.896388:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701435.896393:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701435.896396:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502f00 00000400:00000200:1.0:1790701435.896402:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701435.896408:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x679 [1] + 224 00000400:00000200:1.0:1790701435.896425:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701435.896429:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701435.896443:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b95500 x1877686323523328/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701451 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701435.896809:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823ec8. 00000400:00000200:3.0:1790701435.896815:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee088 00000400:00000010:3.0:1790701435.896818:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee088. 00000100:00000200:3.0:1790701435.896825:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b95500 x1877686323523328/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701451 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701435.896835:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b95500 x1877686323523328/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701451 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701435.896862:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a176c00. 02000000:00000010:3.0:1790701435.896869:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1c00. 00000100:00000010:3.0:1790701435.896874:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95500. 00000100:00000010:1.0:1790701436.844498:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305cba0. 00000100:00000010:0.0:1790701436.844669:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305c540. 00000001:02000400:2.0:1790701436.881862:0:130336:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 603: lnet_legacy2yaml: default hop flag ========================================================== 13:03:55 (1790701435) 00000100:00000010:2.0:1790701441.004250:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94380. 02000000:00000010:2.0:1790701441.004270:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1300. 00000100:00000010:2.0:1790701441.004296:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b97b80. 02000000:00000010:2.0:1790701441.004301:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1400. 02000000:00000010:3.0:1790701441.005317:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf6600. 00000400:00000200:3.0:1790701441.005330:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701441.005334:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701441.005341:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701441.005348:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323523584, portal 25 00000400:00000010:3.0:1790701441.005353:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35908. 00000100:00000200:3.0:1790701441.005356:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323523584, offset 0 00000400:00000200:3.0:1790701441.005365:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701441.005372:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701441.005379:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701441.005392:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701441.005396:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701441.005400:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701441.005403:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701441.005407:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [100:100:256:4294967295] 192.168.206.116@tcp LPNI seq info [100:100:8:4294967295] 00000400:00000200:3.0:1790701441.005411:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701441.005422:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701441.005430:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701441.005435:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f000. 00000800:00000200:3.0:1790701441.005442:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701441.005449:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701441.005453:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701441.005468:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701441.005473:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec33c0. 00000100:00000010:3.0:1790701441.005483:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3660. 02000000:00000010:3.0:1790701441.005504:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf7c00. 00000400:00000200:3.0:1790701441.005508:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701441.005511:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9f20. 00000400:00000010:3.0:1790701441.005514:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35e58. 00000100:00000200:3.0:1790701441.005517:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323523712, portal 10 00000400:00000010:3.0:1790701441.005523:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35220. 00000100:00000200:3.0:1790701441.005525:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323523712, offset 0 00000400:00000200:3.0:1790701441.005529:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701441.005532:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701441.005536:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701441.005538:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701441.005540:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701441.005542:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701441.005544:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701441.005546:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [101:101:255:4294967295] 192.168.206.116@tcp LPNI seq info [101:101:7:4294967295] 00000400:00000200:3.0:1790701441.005550:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701441.005562:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701441.005566:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701441.005570:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f200. 00000800:00000200:3.0:1790701441.005573:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701441.005578:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701441.005581:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701441.005954:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701441.010443:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701441.010452:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f000. 00000400:00000200:1.0:1790701441.010459:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701441.010467:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701441.010474:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35908 00000400:00000010:1.0:1790701441.010477:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35908. 00000100:00000200:1.0:1790701441.010487:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b94380 x1877686323523584/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701457 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701441.010560:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701441.010563:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f200. 00000400:00000200:1.0:1790701441.010566:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701441.010569:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701441.010572:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35220 00000400:00000010:1.0:1790701441.010574:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35220. 00000100:00000200:1.0:1790701441.010578:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b97b80 x1877686323523712/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701457 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701441.017186:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701441.017196:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701441.017205:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701441.017211:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701441.017215:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701441.017219:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503080 00000400:00000200:1.0:1790701441.017224:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701441.017230:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x689 [1] + 224 00000400:00000200:1.0:1790701441.017242:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701441.017247:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701441.017261:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b97b80 x1877686323523712/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701457 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701441.017288:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701441.017292:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701441.017297:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701441.017312:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701441.017316:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701441.017318:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503000 00000400:00000200:1.0:1790701441.017322:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701441.017327:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x681 [1] + 224 00000400:00000200:1.0:1790701441.017337:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701441.017340:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701441.017346:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b94380 x1877686323523584/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701457 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701441.017465:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9f20. 00000400:00000200:3.0:1790701441.017470:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35e58 00000400:00000010:3.0:1790701441.017472:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35e58. 00000100:00000200:3.0:1790701441.017479:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b97b80 x1877686323523712/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701457 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701441.017489:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b97b80 x1877686323523712/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701457 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701441.017515:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:3.0:1790701441.017518:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198 00000400:00000010:3.0:1790701441.017519:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701441.017523:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b94380 x1877686323523584/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701457 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701441.017530:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b94380 x1877686323523584/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701457 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701441.017543:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf7c00. 02000000:00000010:3.0:1790701441.017549:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1400. 00000100:00000010:3.0:1790701441.017552:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b97b80. 02000000:00000010:3.0:1790701441.017556:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf6600. 02000000:00000010:3.0:1790701441.017558:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1300. 00000100:00000010:3.0:1790701441.017560:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94380. 00000100:00000010:0.0:1790701441.969929:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec33c0. 00000100:00000010:1.0:1790701441.970288:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3660. 00000001:02000400:0.0:1790701443.595067:0:130908:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 604: lnet_legacy2yaml: default hop with explicit value ========================================================== 13:04:02 (1790701442) 00000100:00000010:2.0:1790701446.124448:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b96d80. 02000000:00000010:2.0:1790701446.124469:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1900. 00000100:00000010:2.0:1790701446.124504:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b96a00. 02000000:00000010:2.0:1790701446.124510:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1e00. 02000000:00000010:3.0:1790701446.124579:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908445e00. 00000400:00000200:3.0:1790701446.124590:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701446.124606:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701446.124613:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701446.124620:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323523840, portal 25 00000400:00000010:3.0:1790701446.124625:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35e58. 00000100:00000200:3.0:1790701446.124630:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323523840, offset 0 00000400:00000200:3.0:1790701446.124638:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701446.124646:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701446.124653:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701446.124658:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701446.124663:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701446.124666:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701446.124669:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701446.124673:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [102:102:256:4294967295] 192.168.206.116@tcp LPNI seq info [102:102:8:4294967295] 00000400:00000200:3.0:1790701446.124677:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701446.124685:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701446.124692:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701446.124697:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f900. 00000800:00000200:3.0:1790701446.124704:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701446.124712:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701446.124717:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701446.124741:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701446.124746:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3420. 00000100:00000010:3.0:1790701446.124759:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3780. 02000000:00000010:3.0:1790701446.124780:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908444c00. 00000400:00000200:3.0:1790701446.124784:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701446.124786:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9f20. 00000400:00000010:3.0:1790701446.124789:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35ee0. 00000100:00000200:3.0:1790701446.124792:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323523968, portal 10 00000400:00000010:3.0:1790701446.124797:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35b28. 00000100:00000200:3.0:1790701446.124799:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323523968, offset 0 00000400:00000200:3.0:1790701446.124803:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701446.124813:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701446.124816:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701446.124819:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701446.124821:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701446.124824:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701446.124826:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701446.124828:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [103:103:255:4294967295] 192.168.206.116@tcp LPNI seq info [103:103:7:4294967295] 00000400:00000200:3.0:1790701446.124832:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701446.124839:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701446.124843:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701446.124847:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57ff00. 00000800:00000200:3.0:1790701446.124850:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701446.124854:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701446.124856:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57ff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701446.124862:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701446.124943:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701446.124948:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f900. 00000400:00000200:0.0:1790701446.124952:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701446.124957:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701446.124962:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35e58 00000400:00000010:0.0:1790701446.124970:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35e58. 00000100:00000200:0.0:1790701446.124979:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b96d80 x1877686323523840/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701446.125035:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701446.125038:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57ff00. 00000400:00000200:0.0:1790701446.125049:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701446.125053:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701446.125056:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35b28 00000400:00000010:0.0:1790701446.125057:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35b28. 00000100:00000200:0.0:1790701446.125062:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b96a00 x1877686323523968/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701446.128198:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701446.128208:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701446.128217:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701446.128222:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701446.128228:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701446.128232:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503180 00000400:00000200:0.0:1790701446.128237:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701446.128242:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x699 [1] + 224 00000400:00000200:0.0:1790701446.128256:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701446.128261:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701446.128273:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b96a00 x1877686323523968/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701462 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701446.128671:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9f20. 00000400:00000200:3.0:1790701446.128677:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35ee0 00000400:00000010:3.0:1790701446.128679:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35ee0. 00000100:00000200:3.0:1790701446.128685:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b96a00 x1877686323523968/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701462 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701446.128695:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b96a00 x1877686323523968/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701462 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701446.128728:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908444c00. 02000000:00000010:3.0:1790701446.128734:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1e00. 00000100:00000010:3.0:1790701446.128738:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b96a00. 00000400:00000200:0.0:1790701446.130628:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701446.130636:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701446.130645:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701446.130649:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701446.130654:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701446.130658:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503100 00000400:00000200:0.0:1790701446.130663:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701446.130684:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x691 [1] + 224 00000400:00000200:0.0:1790701446.130698:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701446.130702:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701446.130715:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b96d80 x1877686323523840/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701462 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701446.130938:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:3.0:1790701446.130943:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198 00000400:00000010:3.0:1790701446.130946:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701446.130952:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b96d80 x1877686323523840/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701462 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701446.130972:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b96d80 x1877686323523840/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701462 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701446.131006:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908445e00. 02000000:00000010:3.0:1790701446.131012:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1900. 00000100:00000010:3.0:1790701446.131016:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b96d80. 00000100:00000010:1.0:1790701447.084657:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3780. 00000100:00000010:0.0:1790701447.086247:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3420. 00000100:00000010:2.0:1790701451.244270:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94a80. 02000000:00000010:2.0:1790701451.244290:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1a00. 00000100:00000010:2.0:1790701451.244321:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95c00. 02000000:00000010:2.0:1790701451.244325:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1300. 02000000:00000010:2.0:1790701451.244361:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c907cdcc00. 00000400:00000200:2.0:1790701451.244370:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701451.244375:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823e18. 00000400:00000010:2.0:1790701451.244383:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee7f8. 00000100:00000200:2.0:1790701451.244391:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323524096, portal 25 00000400:00000010:2.0:1790701451.244396:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeed48. 00000100:00000200:2.0:1790701451.244399:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323524096, offset 0 00000400:00000200:2.0:1790701451.244408:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701451.244417:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701451.244425:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701451.244430:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701451.244434:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701451.244438:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701451.244449:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701451.244453:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [104:104:256:4294967295] 192.168.206.116@tcp LPNI seq info [104:104:8:4294967295] 00000400:00000200:2.0:1790701451.244457:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701451.244468:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701451.244474:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701451.244479:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1400. 00000800:00000200:2.0:1790701451.244487:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701451.244494:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701451.244498:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701451.244521:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:2.0:1790701451.244526:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305cd80. 00000100:00000010:2.0:1790701451.244536:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305c2a0. 02000000:00000010:2.0:1790701451.244555:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c907cdc400. 00000400:00000200:2.0:1790701451.244559:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701451.244562:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823420. 00000400:00000010:2.0:1790701451.244565:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee2a8. 00000100:00000200:2.0:1790701451.244568:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323524224, portal 10 00000400:00000010:2.0:1790701451.244574:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee330. 00000100:00000200:2.0:1790701451.244576:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323524224, offset 0 00000400:00000200:2.0:1790701451.244580:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701451.244583:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701451.244586:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701451.244589:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701451.244591:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701451.244593:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701451.244595:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701451.244597:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [105:105:255:4294967295] 192.168.206.116@tcp LPNI seq info [105:105:7:4294967295] 00000400:00000200:2.0:1790701451.244601:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701451.244606:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701451.244610:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701451.244619:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1c00. 00000800:00000200:2.0:1790701451.244623:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701451.244628:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701451.244630:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701451.244636:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701451.246246:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701451.246259:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1400. 00000400:00000200:0.0:1790701451.246268:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701451.246277:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701451.246286:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeed48 00000400:00000010:0.0:1790701451.246290:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeed48. 00000100:00000200:0.0:1790701451.246302:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b94a80 x1877686323524096/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701451.246396:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701451.246399:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1c00. 00000400:00000200:0.0:1790701451.246403:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701451.246406:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701451.246409:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee330 00000400:00000010:0.0:1790701451.246411:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee330. 00000100:00000200:0.0:1790701451.246416:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b95c00 x1877686323524224/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701451.254730:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701451.254747:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701451.254757:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701451.254762:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701451.254766:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701451.254770:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503280 00000400:00000200:0.0:1790701451.254776:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701451.254782:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x6a9 [1] + 224 00000400:00000200:0.0:1790701451.254799:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701451.254818:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701451.254831:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b95c00 x1877686323524224/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701467 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701451.255321:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823420. 00000400:00000200:3.0:1790701451.255327:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee2a8 00000400:00000010:3.0:1790701451.255330:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee2a8. 00000100:00000200:3.0:1790701451.255336:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b95c00 x1877686323524224/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701467 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701451.255346:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b95c00 x1877686323524224/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701467 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701451.255379:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c907cdc400. 02000000:00000010:3.0:1790701451.255385:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1300. 00000100:00000010:3.0:1790701451.255389:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95c00. 00000400:00000200:0.0:1790701451.256628:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701451.256635:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701451.256642:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701451.256647:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701451.256652:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701451.256655:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503200 00000400:00000200:0.0:1790701451.256660:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701451.256666:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x6a1 [1] + 224 00000400:00000200:0.0:1790701451.256678:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701451.256682:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701451.256693:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b94a80 x1877686323524096/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701467 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701451.256828:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823e18. 00000400:00000200:3.0:1790701451.256834:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee7f8 00000400:00000010:3.0:1790701451.256836:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee7f8. 00000100:00000200:3.0:1790701451.256842:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b94a80 x1877686323524096/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701467 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701451.256851:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b94a80 x1877686323524096/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701467 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701451.256868:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c907cdcc00. 02000000:00000010:3.0:1790701451.256873:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1a00. 00000100:00000010:3.0:1790701451.256877:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94a80. 00000001:02000400:0.0:1790701452.023495:0:131480:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 605: lnet_legacy2yaml: quote leading star in IP pattern ========================================================== 13:04:10 (1790701450) 00000100:00000010:0.0:1790701452.206873:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305cd80. 00000100:00000010:1.0:1790701452.209808:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305c2a0. 00000100:00000010:2.0:1790701456.366551:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95f80. 02000000:00000010:2.0:1790701456.366573:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1800. 00000100:00000010:2.0:1790701456.366609:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b96680. 02000000:00000010:2.0:1790701456.366614:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1d00. 02000000:00000010:2.0:1790701456.366658:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a177c00. 00000400:00000200:2.0:1790701456.366669:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701456.366673:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c9188234d0. 00000400:00000010:2.0:1790701456.366681:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee440. 00000100:00000200:2.0:1790701456.366689:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323524480, portal 10 00000400:00000010:2.0:1790701456.366697:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee770. 00000100:00000200:2.0:1790701456.366701:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323524480, offset 0 00000400:00000200:2.0:1790701456.366711:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701456.366718:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701456.366726:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701456.366731:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701456.366735:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701456.366739:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701456.366741:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701456.366745:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [106:106:256:4294967295] 192.168.206.116@tcp LPNI seq info [106:106:8:4294967295] 00000400:00000200:2.0:1790701456.366750:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701456.366760:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701456.366767:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701456.366772:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1a00. 00000800:00000200:2.0:1790701456.366779:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701456.366787:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701456.366792:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701456.366821:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:2.0:1790701456.366826:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305cde0. 00000100:00000010:2.0:1790701456.366837:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305c5a0. 02000000:00000010:2.0:1790701456.366857:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a177400. 00000400:00000200:2.0:1790701456.366861:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701456.366864:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823630. 00000400:00000010:2.0:1790701456.366868:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee198. 00000100:00000200:2.0:1790701456.366871:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323524352, portal 25 00000400:00000010:2.0:1790701456.366875:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee000. 00000100:00000200:2.0:1790701456.366877:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323524352, offset 0 00000400:00000200:2.0:1790701456.366882:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701456.366885:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701456.366888:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701456.366891:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701456.366893:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701456.366895:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701456.366897:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701456.366900:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [107:107:255:4294967295] 192.168.206.116@tcp LPNI seq info [107:107:7:4294967295] 00000400:00000200:2.0:1790701456.366903:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701456.366909:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701456.366913:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701456.366916:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1300. 00000800:00000200:2.0:1790701456.366919:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701456.366923:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701456.366926:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701456.366930:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701456.367115:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701456.367120:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1a00. 00000400:00000200:0.0:1790701456.367125:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701456.367132:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701456.367138:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee770 00000400:00000010:0.0:1790701456.367140:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee770. 00000100:00000200:0.0:1790701456.367150:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b96680 x1877686323524480/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701456.367251:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701456.367254:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1300. 00000400:00000200:0.0:1790701456.367258:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701456.367261:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701456.367271:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee000 00000400:00000010:0.0:1790701456.367273:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee000. 00000100:00000200:0.0:1790701456.367279:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b95f80 x1877686323524352/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701456.371221:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701456.371235:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701456.371247:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701456.371253:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701456.371259:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701456.371264:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503300 00000400:00000200:1.0:1790701456.371273:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701456.371282:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x6b9 [1] + 224 00000400:00000200:1.0:1790701456.371297:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701456.371303:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701456.371322:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b95f80 x1877686323524352/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701472 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701456.371363:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701456.371371:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701456.371376:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701456.371378:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701456.371381:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701456.371388:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503380 00000400:00000200:1.0:1790701456.371404:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701456.371409:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x6b1 [1] + 224 00000400:00000200:1.0:1790701456.371427:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701456.371431:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701456.371438:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b96680 x1877686323524480/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701472 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701456.371546:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823630. 00000400:00000200:2.0:1790701456.371552:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee198 00000400:00000010:2.0:1790701456.371554:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee198. 00000100:00000200:2.0:1790701456.371561:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b95f80 x1877686323524352/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701472 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701456.371583:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b95f80 x1877686323524352/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701472 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701456.371610:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c9188234d0. 00000400:00000200:2.0:1790701456.371612:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee440 00000400:00000010:2.0:1790701456.371614:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee440. 00000100:00000200:2.0:1790701456.371619:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b96680 x1877686323524480/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701472 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701456.371630:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b96680 x1877686323524480/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701472 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701456.371663:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a177400. 02000000:00000010:2.0:1790701456.371672:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1800. 00000100:00000010:2.0:1790701456.371677:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95f80. 02000000:00000010:2.0:1790701456.371681:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a177c00. 02000000:00000010:2.0:1790701456.371683:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1d00. 00000100:00000010:2.0:1790701456.371685:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b96680. 00000100:00000010:1.0:1790701457.324266:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305c5a0. 00000100:00000010:0.0:1790701457.324266:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305cde0. 00000001:02000400:1.0:1790701459.075420:0:132052:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 606: lnet_legacy2yaml: file inputs === 13:04:17 (1790701457) 00000100:00000010:2.0:1790701461.484255:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b96680. 02000000:00000010:2.0:1790701461.484273:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1900. 00000100:00000010:2.0:1790701461.484298:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95f80. 02000000:00000010:2.0:1790701461.484303:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1400. 02000000:00000010:2.0:1790701461.484401:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91885cc00. 00000400:00000200:2.0:1790701461.484410:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701461.484413:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c9188234d0. 00000400:00000010:2.0:1790701461.484419:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee440. 00000100:00000200:2.0:1790701461.484426:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323524608, portal 25 00000400:00000010:2.0:1790701461.484430:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee198. 00000100:00000200:2.0:1790701461.484433:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323524608, offset 0 00000400:00000200:2.0:1790701461.484442:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701461.484449:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701461.484456:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701461.484468:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701461.484473:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701461.484476:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701461.484479:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701461.484482:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [108:108:256:4294967295] 192.168.206.116@tcp LPNI seq info [108:108:8:4294967295] 00000400:00000200:2.0:1790701461.484487:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701461.484496:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701461.484501:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701461.484505:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1c00. 00000800:00000200:2.0:1790701461.484512:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701461.484518:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701461.484522:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701461.484531:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:2.0:1790701461.484535:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305cf60. 00000100:00000010:2.0:1790701461.484545:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305c480. 02000000:00000010:2.0:1790701461.484562:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91885ce00. 00000400:00000200:2.0:1790701461.484565:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701461.484568:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823630. 00000400:00000010:2.0:1790701461.484571:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeebb0. 00000100:00000200:2.0:1790701461.484573:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323524736, portal 10 00000400:00000010:2.0:1790701461.484578:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeef68. 00000100:00000200:2.0:1790701461.484580:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323524736, offset 0 00000400:00000200:2.0:1790701461.484584:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701461.484587:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701461.484590:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701461.484593:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701461.484595:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701461.484597:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701461.484599:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701461.484602:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [109:109:255:4294967295] 192.168.206.116@tcp LPNI seq info [109:109:7:4294967295] 00000400:00000200:2.0:1790701461.484605:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701461.484616:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701461.484621:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701461.484623:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1300. 00000800:00000200:2.0:1790701461.484627:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701461.484631:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701461.484633:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701461.484639:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701461.484822:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701461.484826:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1c00. 00000400:00000200:0.0:1790701461.484843:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701461.484848:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701461.484854:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee198 00000400:00000010:0.0:1790701461.484856:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee198. 00000100:00000200:0.0:1790701461.484864:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b96680 x1877686323524608/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701461.484941:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701461.484944:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1300. 00000400:00000200:0.0:1790701461.484947:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701461.484950:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701461.484953:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeef68 00000400:00000010:0.0:1790701461.484954:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeef68. 00000100:00000200:0.0:1790701461.484959:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b95f80 x1877686323524736/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701461.493814:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701461.493824:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701461.493832:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701461.493838:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701461.493843:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701461.493847:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503480 00000400:00000200:1.0:1790701461.493852:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701461.493859:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x6c9 [1] + 224 00000400:00000200:1.0:1790701461.493872:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701461.493876:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701461.493891:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b95f80 x1877686323524736/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701477 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701461.493957:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823630. 00000400:00000200:3.0:1790701461.493963:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeebb0 00000400:00000010:3.0:1790701461.493966:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeebb0. 00000100:00000200:3.0:1790701461.493972:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b95f80 x1877686323524736/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701477 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701461.493993:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b95f80 x1877686323524736/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701477 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701461.494035:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91885ce00. 02000000:00000010:3.0:1790701461.494040:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1400. 00000100:00000010:3.0:1790701461.494044:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95f80. 00000400:00000200:0.0:1790701461.496336:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701461.496346:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701461.496354:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701461.496359:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701461.496364:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701461.496367:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503400 00000400:00000200:0.0:1790701461.496372:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701461.496379:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x6c1 [1] + 224 00000400:00000200:1.0:1790701461.496541:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701461.496548:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701461.496561:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b96680 x1877686323524608/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701477 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701461.496786:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c9188234d0. 00000400:00000200:3.0:1790701461.496791:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee440 00000400:00000010:3.0:1790701461.496794:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee440. 00000100:00000200:3.0:1790701461.496801:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b96680 x1877686323524608/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701477 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701461.496809:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b96680 x1877686323524608/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701477 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701461.496831:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91885cc00. 02000000:00000010:3.0:1790701461.496835:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1900. 00000100:00000010:3.0:1790701461.496840:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b96680. 00000100:00000010:0.0:1790701462.444331:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305cf60. 00000100:00000010:1.0:1790701462.445216:0:91773:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305c480. 00000100:00000010:3.0:1790701466.604345:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908dae300. 02000000:00000010:3.0:1790701466.604365:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d57f400. 00000100:00000010:3.0:1790701466.604408:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908daf100. 02000000:00000010:3.0:1790701466.604413:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d57f900. 02000000:00000010:3.0:1790701466.604478:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c924002e00. 00000400:00000200:3.0:1790701466.604488:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701466.604492:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701466.604499:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701466.604506:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323524992, portal 10 00000400:00000010:3.0:1790701466.604512:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35ee0. 00000100:00000200:3.0:1790701466.604517:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323524992, offset 0 00000400:00000200:3.0:1790701466.604526:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701466.604533:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701466.604540:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701466.604551:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701466.604556:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701466.604559:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701466.604561:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701466.604565:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [110:110:256:4294967295] 192.168.206.116@tcp LPNI seq info [110:110:8:4294967295] 00000400:00000200:3.0:1790701466.604569:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701466.604579:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701466.604585:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701466.604589:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57fc00. 00000800:00000200:3.0:1790701466.604595:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701466.604602:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701466.604606:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57fc00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701466.604625:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701466.604629:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3cc0. 00000100:00000010:3.0:1790701466.604640:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3720. 02000000:00000010:3.0:1790701466.604672:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c924003800. 00000400:00000200:3.0:1790701466.604677:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701466.604679:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9f20. 00000400:00000010:3.0:1790701466.604683:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35550. 00000100:00000200:3.0:1790701466.604686:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323524864, portal 25 00000400:00000010:3.0:1790701466.604689:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d356e8. 00000100:00000200:3.0:1790701466.604691:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323524864, offset 0 00000400:00000200:3.0:1790701466.604696:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701466.604700:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701466.604703:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701466.604706:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701466.604709:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701466.604711:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701466.604714:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701466.604717:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [111:111:255:4294967295] 192.168.206.116@tcp LPNI seq info [111:111:7:4294967295] 00000400:00000200:3.0:1790701466.604721:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701466.604733:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701466.604737:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701466.604740:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f000. 00000800:00000200:3.0:1790701466.604743:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701466.604748:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701466.604750:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701466.604755:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701466.604934:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701466.604938:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57fc00. 00000400:00000200:1.0:1790701466.604943:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701466.604949:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701466.604960:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35ee0 00000400:00000010:1.0:1790701466.604962:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35ee0. 00000100:00000200:1.0:1790701466.604970:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908daf100 x1877686323524992/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701466.605047:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701466.605050:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f000. 00000400:00000200:1.0:1790701466.605053:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701466.605056:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701466.605059:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d356e8 00000400:00000010:1.0:1790701466.605061:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d356e8. 00000100:00000200:1.0:1790701466.605066:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908dae300 x1877686323524864/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701466.610516:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701466.610526:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701466.610536:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701466.610541:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701466.610546:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701466.610549:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503500 00000400:00000200:1.0:1790701466.610554:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701466.610560:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x6d9 [1] + 224 00000400:00000200:1.0:1790701466.610573:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701466.610577:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701466.610591:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908dae300 x1877686323524864/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701482 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701466.612356:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9f20. 00000400:00000200:3.0:1790701466.612367:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35550 00000400:00000010:3.0:1790701466.612369:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35550. 00000100:00000200:3.0:1790701466.612382:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908dae300 x1877686323524864/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701482 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701466.612394:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908dae300 x1877686323524864/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701482 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701466.612423:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c924003800. 02000000:00000010:3.0:1790701466.612430:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d57f400. 00000100:00000010:3.0:1790701466.612435:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908dae300. 00000400:00000200:0.0:1790701466.613930:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701466.613944:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701466.613952:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701466.613956:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701466.613961:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701466.613964:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503580 00000400:00000200:0.0:1790701466.613969:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701466.613975:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x6d1 [1] + 224 00000400:00000200:0.0:1790701466.613988:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701466.613993:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701466.614015:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908daf100 x1877686323524992/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701482 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701466.615421:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:3.0:1790701466.615429:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198 00000400:00000010:3.0:1790701466.615432:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701466.615442:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908daf100 x1877686323524992/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701482 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701466.615453:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908daf100 x1877686323524992/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701482 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701466.615493:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c924002e00. 02000000:00000010:3.0:1790701466.615498:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d57f900. 00000100:00000010:3.0:1790701466.615502:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908daf100. 00000001:02000400:0.0:1790701467.533696:0:132625:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 607: lnet_legacy2yaml: options line with combined ip2nets and routes ========================================================== 13:04:25 (1790701465) 00000100:00000010:0.0:1790701467.564251:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3cc0. 00000100:00000010:1.0:1790701467.582709:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3720. 00000100:00000010:2.0:1790701471.735547:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b97100. 02000000:00000010:2.0:1790701471.735566:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be79a00. 00000100:00000010:2.0:1790701471.735592:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b96680. 02000000:00000010:2.0:1790701471.735597:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be79c00. 02000000:00000010:3.0:1790701471.736053:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a176e00. 00000400:00000200:3.0:1790701471.736065:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701471.736078:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701471.736086:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701471.736093:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323525120, portal 25 00000400:00000010:3.0:1790701471.736100:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35550. 00000100:00000200:3.0:1790701471.736104:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323525120, offset 0 00000400:00000200:3.0:1790701471.736127:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701471.736137:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701471.736144:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701471.736149:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701471.736189:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701471.736193:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701471.736196:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701471.736201:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [112:112:256:4294967295] 192.168.206.116@tcp LPNI seq info [112:112:8:4294967295] 00000400:00000200:3.0:1790701471.736205:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701471.736217:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701471.736223:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701471.736229:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f900. 00000800:00000200:3.0:1790701471.736236:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701471.736244:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701471.736248:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701471.736267:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701471.736272:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3240. 00000100:00000010:3.0:1790701471.736285:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec39c0. 02000000:00000010:3.0:1790701471.736310:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a177c00. 00000400:00000200:3.0:1790701471.736314:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701471.736317:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9f20. 00000400:00000010:3.0:1790701471.736321:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d356e8. 00000100:00000200:3.0:1790701471.736325:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323525248, portal 10 00000400:00000010:3.0:1790701471.736330:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35ee0. 00000100:00000200:3.0:1790701471.736332:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323525248, offset 0 00000400:00000200:3.0:1790701471.736338:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000800:00000200:1.0:1790701471.736339:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701471.736345:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f900. 00000400:00000200:3.0:1790701471.736350:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.736351:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701471.736356:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701471.736362:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:3.0:1790701471.736364:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701471.736369:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701471.736369:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35550 00000400:00000010:1.0:1790701471.736374:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35550. 00000400:00000200:3.0:1790701471.736375:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701471.736378:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701471.736380:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [113:113:255:4294967295] 192.168.206.116@tcp LPNI seq info [113:113:7:4294967295] 00000400:00000200:3.0:1790701471.736386:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000200:1.0:1790701471.736392:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b97100 x1877686323525120/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701471.736397:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701471.736408:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701471.736414:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f400. 00000800:00000200:3.0:1790701471.736418:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701471.736423:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701471.736426:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701471.736460:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701471.741621:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701471.741642:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f400. 00000400:00000200:1.0:1790701471.741648:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701471.741656:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701471.741664:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35ee0 00000400:00000010:1.0:1790701471.741666:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35ee0. 00000100:00000200:1.0:1790701471.741678:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b96680 x1877686323525248/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701471.747949:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.747960:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701471.747969:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.747975:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.747980:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.747984:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503600 00000400:00000200:1.0:1790701471.747990:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.747995:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x6e1 [1] + 224 00000400:00000200:1.0:1790701471.748010:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701471.748014:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701471.748027:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b97100 x1877686323525120/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701487 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701471.750791:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:3.0:1790701471.750879:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198 00000400:00000010:3.0:1790701471.750882:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701471.750895:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b97100 x1877686323525120/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701487 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701471.750906:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b97100 x1877686323525120/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701487 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701471.750937:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a176e00. 02000000:00000010:3.0:1790701471.750943:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be79a00. 00000100:00000010:3.0:1790701471.750948:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b97100. 00000400:00000200:1.0:1790701471.757556:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.757565:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701471.757574:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.757578:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.757583:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.757587:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503680 00000400:00000200:1.0:1790701471.757592:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701471.757597:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x6e9 [1] + 224 00000400:00000200:1.0:1790701471.757618:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701471.757623:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701471.757637:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b96680 x1877686323525248/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701487 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701471.757740:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9f20. 00000400:00000200:3.0:1790701471.757745:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d356e8 00000400:00000010:3.0:1790701471.757747:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d356e8. 00000100:00000200:3.0:1790701471.757754:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b96680 x1877686323525248/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701487 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701471.757774:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b96680 x1877686323525248/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701487 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701471.757815:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a177c00. 02000000:00000010:3.0:1790701471.757821:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be79c00. 00000100:00000010:3.0:1790701471.757826:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b96680. 00000100:00000010:1.0:1790701472.684335:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec39c0. 00000100:00000010:0.0:1790701472.685952:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3240. 00000001:02000400:0.0:1790701475.739974:0:133198:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 608: lnet_legacy2yaml: error on invalid rule (missing IP pattern) ========================================================== 13:04:34 (1790701474) 00000100:00000010:1.0:1790701476.845274:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd2300. 02000000:00000010:1.0:1790701476.845429:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bf7e000. 00000100:00000010:1.0:1790701476.845461:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd1c00. 02000000:00000010:1.0:1790701476.845465:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bf7e500. 02000000:00000010:1.0:1790701476.845504:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c93bc23000. 00000400:00000200:1.0:1790701476.845514:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701476.845519:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69580. 00000400:00000010:1.0:1790701476.845527:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bdd0. 00000100:00000200:1.0:1790701476.845535:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323525376, portal 25 00000400:00000010:1.0:1790701476.845541:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bee0. 00000100:00000200:1.0:1790701476.845543:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323525376, offset 0 00000400:00000200:1.0:1790701476.845551:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701476.845558:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.845566:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701476.845571:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701476.845574:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701476.845578:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701476.845580:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701476.845583:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [114:114:256:4294967295] 192.168.206.116@tcp LPNI seq info [114:114:8:4294967295] 00000400:00000200:1.0:1790701476.845587:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.845595:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701476.845600:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701476.845604:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bf7e600. 00000800:00000200:1.0:1790701476.845611:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701476.845618:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701476.845628:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bf7e600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701476.845640:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:1.0:1790701476.845646:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac9c0. 00000100:00000010:1.0:1790701476.845657:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac0c0. 02000000:00000010:1.0:1790701476.845675:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c93bc22600. 00000400:00000200:1.0:1790701476.845679:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701476.845759:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69318. 00000400:00000010:1.0:1790701476.845765:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b220. 00000100:00000200:1.0:1790701476.845768:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323525504, portal 10 00000400:00000010:1.0:1790701476.845775:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701476.845777:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323525504, offset 0 00000400:00000200:1.0:1790701476.845863:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701476.845869:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.845873:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701476.845875:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701476.845878:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701476.845880:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701476.845882:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701476.845885:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [115:115:255:4294967295] 192.168.206.116@tcp LPNI seq info [115:115:7:4294967295] 00000400:00000200:1.0:1790701476.845888:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.845893:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701476.845897:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701476.845900:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bf7e300. 00000800:00000200:1.0:1790701476.845904:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701476.845908:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701476.845910:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bf7e300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701476.845916:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701476.845964:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701476.845968:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bf7e600. 00000400:00000200:1.0:1790701476.845973:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701476.845977:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701476.845988:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bee0 00000400:00000010:1.0:1790701476.845991:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bee0. 00000100:00000200:1.0:1790701476.845999:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910bd2300 x1877686323525376/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701476.848709:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701476.848717:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bf7e300. 00000400:00000200:1.0:1790701476.848725:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701476.848733:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701476.848739:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b908 00000400:00000010:1.0:1790701476.848741:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701476.848751:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910bd1c00 x1877686323525504/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701476.862029:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.862039:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701476.862047:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.862051:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.862056:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.862060:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503700 00000400:00000200:1.0:1790701476.862065:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.862070:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x6f1 [1] + 224 00000400:00000200:1.0:1790701476.862081:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701476.862085:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701476.862098:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910bd2300 x1877686323525376/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701492 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701476.862135:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69580. 00000400:00000200:1.0:1790701476.862140:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bdd0 00000400:00000010:1.0:1790701476.862142:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bdd0. 00000100:00000200:1.0:1790701476.862148:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910bd2300 x1877686323525376/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701492 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701476.862367:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910bd2300 x1877686323525376/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701492 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701476.862399:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c93bc23000. 02000000:00000010:1.0:1790701476.863361:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bf7e000. 00000100:00000010:1.0:1790701476.863370:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd2300. 00000400:00000200:1.0:1790701476.863400:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.863475:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701476.863483:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.863487:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.863491:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.863494:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503780 00000400:00000200:1.0:1790701476.863498:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701476.863504:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x701 [1] + 224 00000400:00000200:1.0:1790701476.863516:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701476.863520:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701476.863530:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910bd1c00 x1877686323525504/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701492 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701476.863562:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69318. 00000400:00000200:1.0:1790701476.863565:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b220 00000400:00000010:1.0:1790701476.863567:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b220. 00000100:00000200:1.0:1790701476.863572:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910bd1c00 x1877686323525504/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701492 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701476.863579:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910bd1c00 x1877686323525504/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701492 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701476.863601:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c93bc22600. 02000000:00000010:1.0:1790701476.863606:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bf7e500. 00000100:00000010:1.0:1790701476.863608:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd1c00. 00000100:00000010:1.0:1790701477.811842:0:91773:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac0c0. 00000100:00000010:0.0:1790701477.813852:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac9c0. 00000100:00000010:2.0:1790701481.966535:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95f80. 02000000:00000010:2.0:1790701481.966574:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be79b00. 00000100:00000010:2.0:1790701481.966612:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94a80. 02000000:00000010:2.0:1790701481.966618:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be79400. 02000000:00000010:2.0:1790701481.966656:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c924002c00. 00000400:00000200:2.0:1790701481.966668:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701481.966674:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c9188235d8. 00000400:00000010:2.0:1790701481.966682:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee990. 00000100:00000200:2.0:1790701481.966691:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323525760, portal 10 00000400:00000010:2.0:1790701481.966701:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee440. 00000100:00000200:2.0:1790701481.966705:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323525760, offset 0 00000400:00000200:2.0:1790701481.966715:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701481.966725:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701481.966735:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701481.966740:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701481.966984:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701481.966989:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701481.966993:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701481.966997:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [116:116:256:4294967295] 192.168.206.116@tcp LPNI seq info [116:116:8:4294967295] 00000400:00000200:2.0:1790701481.967002:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701481.967028:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701481.967035:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701481.967041:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be79d00. 00000800:00000200:2.0:1790701481.967048:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701481.967058:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701481.967063:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be79d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701481.967084:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:2.0:1790701481.967090:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305c120. 00000100:00000010:2.0:1790701481.967103:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305ca80. 02000000:00000010:2.0:1790701481.967127:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c924003a00. 00000400:00000200:2.0:1790701481.967132:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701481.967135:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823c60. 00000400:00000010:2.0:1790701481.967139:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeebb0. 00000100:00000200:2.0:1790701481.967142:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323525632, portal 25 00000400:00000010:2.0:1790701481.967146:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeef68. 00000100:00000200:2.0:1790701481.967149:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323525632, offset 0 00000400:00000200:2.0:1790701481.967175:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701481.967179:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701481.967182:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701481.967185:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701481.967188:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701481.967190:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701481.967192:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701481.967195:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [117:117:255:4294967295] 192.168.206.116@tcp LPNI seq info [117:117:7:4294967295] 00000400:00000200:2.0:1790701481.967198:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701481.967204:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701481.967208:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701481.968006:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be7f000. 00000800:00000200:2.0:1790701481.968016:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701481.968022:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701481.968037:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be7f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701481.968046:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701481.969355:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701481.969364:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be79d00. 00000400:00000200:0.0:1790701481.969423:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701481.969432:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701481.969440:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee440 00000400:00000010:0.0:1790701481.969443:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee440. 00000100:00000200:0.0:1790701481.969454:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b94a80 x1877686323525760/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701481.969542:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701481.969544:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be7f000. 00000400:00000200:0.0:1790701481.969548:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701481.969552:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701481.969555:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeef68 00000400:00000010:0.0:1790701481.969565:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeef68. 00000100:00000200:0.0:1790701481.969572:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b95f80 x1877686323525632/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701481.971931:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701481.971943:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701481.971952:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701481.971959:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701481.971965:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701481.971969:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503880 00000400:00000200:0.0:1790701481.971977:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701481.971984:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x711 [1] + 224 00000400:00000200:0.0:1790701481.972000:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701481.972005:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701481.972020:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b94a80 x1877686323525760/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701497 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701481.972325:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c9188235d8. 00000400:00000200:2.0:1790701481.972331:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee990 00000400:00000010:2.0:1790701481.972334:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee990. 00000100:00000200:2.0:1790701481.972342:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b94a80 x1877686323525760/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701497 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701481.972351:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b94a80 x1877686323525760/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701497 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701481.972844:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c924002c00. 02000000:00000010:2.0:1790701481.972853:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be79400. 00000100:00000010:2.0:1790701481.972858:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94a80. 00000400:00000200:0.0:1790701481.976084:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701481.976102:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701481.976124:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701481.976131:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701481.976139:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701481.976144:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503800 00000400:00000200:0.0:1790701481.976150:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701481.976192:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x719 [1] + 224 00000400:00000200:0.0:1790701481.977506:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701481.977520:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701481.977539:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b95f80 x1877686323525632/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701497 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701481.978549:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823c60. 00000400:00000200:2.0:1790701481.978560:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeebb0 00000400:00000010:2.0:1790701481.978564:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeebb0. 00000100:00000200:2.0:1790701481.978574:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b95f80 x1877686323525632/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701497 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701481.978586:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b95f80 x1877686323525632/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701497 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701481.978626:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c924003a00. 02000000:00000010:2.0:1790701481.978638:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be79b00. 00000100:00000010:2.0:1790701481.978644:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95f80. 00000100:00000010:0.0:1790701482.924280:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305c120. 00000100:00000010:1.0:1790701482.925359:0:91773:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305ca80. 00000001:02000400:2.0:1790701484.606971:0:133769:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 609: lnet_legacy2yaml: error on no input ========================================================== 13:04:42 (1790701482) 00000100:00000010:3.0:1790701487.091919:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908daf100. 02000000:00000010:3.0:1790701487.091950:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d57fe00. 00000100:00000010:3.0:1790701487.091988:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908dae300. 02000000:00000010:3.0:1790701487.091993:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d57fa00. 02000000:00000010:3.0:1790701487.092037:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c907a79e00. 00000400:00000200:3.0:1790701487.092050:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701487.092056:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9f20. 00000400:00000010:3.0:1790701487.092064:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d356e8. 00000100:00000200:3.0:1790701487.092073:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323525888, portal 25 00000400:00000010:3.0:1790701487.092079:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701487.092084:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323525888, offset 0 00000400:00000200:3.0:1790701487.092094:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701487.092104:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701487.092113:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701487.092119:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701487.092124:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701487.092447:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701487.092452:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701487.092456:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [118:118:256:4294967295] 192.168.206.116@tcp LPNI seq info [118:118:8:4294967295] 00000400:00000200:3.0:1790701487.092462:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701487.092476:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701487.092485:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701487.092494:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f600. 00000800:00000200:3.0:1790701487.092503:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701487.092530:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701487.092535:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701487.092554:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701487.092560:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3180. 00000100:00000010:3.0:1790701487.092578:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3f00. 02000000:00000010:3.0:1790701487.092610:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c907a78400. 00000400:00000200:3.0:1790701487.092615:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701487.092619:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701487.092626:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35b28. 00000100:00000200:3.0:1790701487.092630:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323526016, portal 10 00000400:00000010:3.0:1790701487.092639:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35e58. 00000100:00000200:3.0:1790701487.092643:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323526016, offset 0 00000400:00000200:3.0:1790701487.092650:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701487.092656:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701487.092661:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701487.092665:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701487.092669:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701487.092672:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701487.092675:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701487.092678:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [119:119:255:4294967295] 192.168.206.116@tcp LPNI seq info [119:119:7:4294967295] 00000400:00000200:3.0:1790701487.092682:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701487.092688:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701487.092694:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701487.092699:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f500. 00000800:00000200:3.0:1790701487.092704:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701487.093052:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701487.093055:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701487.093063:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701487.093457:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701487.093471:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f600. 00000400:00000200:0.0:1790701487.093477:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701487.093483:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701487.093492:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198 00000400:00000010:0.0:1790701487.093495:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198. 00000100:00000200:0.0:1790701487.093507:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908daf100 x1877686323525888/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701487.093595:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701487.093598:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f500. 00000400:00000200:0.0:1790701487.093602:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701487.093606:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701487.093609:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35e58 00000400:00000010:0.0:1790701487.093611:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35e58. 00000100:00000200:0.0:1790701487.093616:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908dae300 x1877686323526016/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701487.099038:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701487.099052:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701487.099062:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701487.099067:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701487.099073:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701487.099078:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503900 00000400:00000200:0.0:1790701487.099085:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701487.099091:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x721 [1] + 224 00000400:00000200:0.0:1790701487.099105:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701487.099110:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701487.099126:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908daf100 x1877686323525888/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701503 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701487.099418:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9f20. 00000400:00000200:3.0:1790701487.099426:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d356e8 00000400:00000010:3.0:1790701487.099430:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d356e8. 00000100:00000200:3.0:1790701487.099446:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908daf100 x1877686323525888/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701503 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701487.099455:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908daf100 x1877686323525888/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701503 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701487.099460:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701487.099470:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701487.099475:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701487.099484:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701487.099487:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701487.099491:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503980 00000400:00000200:0.0:1790701487.099495:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 02000000:00000010:3.0:1790701487.099498:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c907a79e00. 00000400:00000200:0.0:1790701487.099502:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x729 [1] + 224 02000000:00000010:3.0:1790701487.099508:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d57fe00. 00000100:00000010:3.0:1790701487.099514:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908daf100. 00000400:00000200:0.0:1790701487.099516:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701487.099522:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701487.099529:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908dae300 x1877686323526016/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701503 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701487.106266:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:3.0:1790701487.106281:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35b28 00000400:00000010:3.0:1790701487.106285:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35b28. 00000100:00000200:3.0:1790701487.106304:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908dae300 x1877686323526016/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701503 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701487.106318:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908dae300 x1877686323526016/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701503 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701487.106894:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c907a78400. 02000000:00000010:3.0:1790701487.106902:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d57fa00. 00000100:00000010:3.0:1790701487.106907:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908dae300. 00000100:00000010:1.0:1790701488.044321:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3f00. 00000100:00000010:0.0:1790701488.044425:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3180. 00000001:02000400:0.0:1790701491.784590:0:134341:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 610: lnet_legacy2yaml: multiple files and inline, check order ========================================================== 13:04:50 (1790701490) 00000100:00000010:1.0:1790701492.205323:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd1c00. 02000000:00000010:1.0:1790701492.205349:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bf7e700. 00000100:00000010:1.0:1790701492.205379:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd2300. 02000000:00000010:1.0:1790701492.205383:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bf7e100. 02000000:00000010:1.0:1790701492.205415:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c93bc23600. 00000400:00000200:1.0:1790701492.205442:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701492.205446:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69318. 00000400:00000010:1.0:1790701492.205453:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b220. 00000100:00000200:1.0:1790701492.205460:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323526272, portal 10 00000400:00000010:1.0:1790701492.205467:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bdd0. 00000100:00000200:1.0:1790701492.205469:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323526272, offset 0 00000400:00000200:1.0:1790701492.205477:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701492.205485:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701492.205492:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701492.205497:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701492.205501:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701492.205504:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701492.205507:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701492.205511:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [120:120:256:4294967295] 192.168.206.116@tcp LPNI seq info [120:120:8:4294967295] 00000400:00000200:1.0:1790701492.205515:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701492.205523:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701492.205528:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701492.205532:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bf7e300. 00000800:00000200:1.0:1790701492.205539:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701492.205545:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701492.205549:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bf7e300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701492.205563:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:1.0:1790701492.205568:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cacae0. 00000100:00000010:1.0:1790701492.205582:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac540. 02000000:00000010:1.0:1790701492.205600:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c93bc22e00. 00000400:00000200:1.0:1790701492.205603:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701492.205612:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69580. 00000400:00000010:1.0:1790701492.205616:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701492.205618:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323526144, portal 25 00000400:00000010:1.0:1790701492.205622:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bee0. 00000100:00000200:1.0:1790701492.205624:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323526144, offset 0 00000400:00000200:1.0:1790701492.205628:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701492.205631:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701492.205635:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701492.205637:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701492.205639:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701492.205642:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701492.205644:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701492.205647:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [121:121:255:4294967295] 192.168.206.116@tcp LPNI seq info [121:121:7:4294967295] 00000400:00000200:1.0:1790701492.205650:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701492.205656:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701492.205660:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701492.205663:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bf7e600. 00000800:00000200:1.0:1790701492.205666:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701492.205670:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701492.205673:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bf7e600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701492.205676:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701492.205720:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701492.205724:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bf7e300. 00000400:00000200:1.0:1790701492.205729:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701492.205733:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701492.205738:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bdd0 00000400:00000010:1.0:1790701492.205740:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bdd0. 00000100:00000200:1.0:1790701492.205749:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910bd2300 x1877686323526272/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701492.205822:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701492.205824:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bf7e600. 00000400:00000200:1.0:1790701492.205831:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701492.205835:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701492.205838:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bee0 00000400:00000010:1.0:1790701492.205840:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bee0. 00000100:00000200:1.0:1790701492.205845:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910bd1c00 x1877686323526144/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701492.211088:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701492.211097:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701492.211107:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701492.211111:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701492.211117:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701492.211121:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503a00 00000400:00000200:0.0:1790701492.211126:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701492.211131:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x741 [1] + 224 00000400:00000200:0.0:1790701492.211141:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701492.211145:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701492.211180:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910bd1c00 x1877686323526144/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701492.211205:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701492.211208:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701492.211212:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701492.211214:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701492.211217:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701492.211219:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503a80 00000400:00000200:0.0:1790701492.211230:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701492.211235:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x731 [1] + 224 00000400:00000200:0.0:1790701492.211246:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701492.211249:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701492.211255:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910bd2300 x1877686323526272/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701492.211973:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69580. 00000400:00000200:1.0:1790701492.211981:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b908 00000400:00000010:1.0:1790701492.211983:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701492.211991:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910bd1c00 x1877686323526144/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701492.211999:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910bd1c00 x1877686323526144/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701508 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701492.212021:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69318. 00000400:00000200:1.0:1790701492.212024:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b220 00000400:00000010:1.0:1790701492.212026:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b220. 00000100:00000200:1.0:1790701492.212030:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910bd2300 x1877686323526272/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701492.212036:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910bd2300 x1877686323526272/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701508 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701492.212055:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c93bc22e00. 02000000:00000010:1.0:1790701492.212061:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bf7e700. 00000100:00000010:1.0:1790701492.212065:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd1c00. 02000000:00000010:1.0:1790701492.212069:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c93bc23600. 02000000:00000010:1.0:1790701492.212071:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bf7e100. 00000100:00000010:1.0:1790701492.212073:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd2300. 00000100:00000010:1.0:1790701493.164301:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac540. 00000100:00000010:0.0:1790701493.170302:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cacae0. 00000100:00000010:1.0:1790701497.324687:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd2300. 02000000:00000010:1.0:1790701497.324710:0:8:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bf7e100. 00000100:00000010:1.0:1790701497.324757:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd1c00. 02000000:00000010:1.0:1790701497.324779:0:8:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bf7e300. 02000000:00000010:1.0:1790701497.324813:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c93bc22e00. 00000400:00000200:1.0:1790701497.324824:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701497.324828:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69318. 00000400:00000010:1.0:1790701497.324835:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b220. 00000100:00000200:1.0:1790701497.324842:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323526528, portal 10 00000400:00000010:1.0:1790701497.324849:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701497.324853:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323526528, offset 0 00000400:00000200:1.0:1790701497.324861:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701497.324869:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.324876:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701497.324881:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701497.324885:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701497.324889:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701497.324892:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701497.324895:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [122:122:256:4294967295] 192.168.206.116@tcp LPNI seq info [122:122:8:4294967295] 00000400:00000200:1.0:1790701497.324900:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.324909:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701497.324914:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701497.324919:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bf7e900. 00000800:00000200:1.0:1790701497.324925:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701497.324932:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701497.324935:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bf7e900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701497.324952:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:1.0:1790701497.324956:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac8a0. 00000100:00000010:1.0:1790701497.324966:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac360. 02000000:00000010:1.0:1790701497.324987:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c93bc23c00. 00000400:00000200:1.0:1790701497.324991:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701497.324993:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69580. 00000400:00000010:1.0:1790701497.324997:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bee0. 00000100:00000200:1.0:1790701497.325000:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323526400, portal 25 00000400:00000010:1.0:1790701497.325010:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bdd0. 00000100:00000200:1.0:1790701497.325012:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323526400, offset 0 00000400:00000200:1.0:1790701497.325017:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701497.325021:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.325025:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701497.325027:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701497.325030:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701497.325032:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701497.325034:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701497.325036:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [123:123:255:4294967295] 192.168.206.116@tcp LPNI seq info [123:123:7:4294967295] 00000400:00000200:1.0:1790701497.325040:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.325045:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701497.325049:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701497.325052:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bf7ee00. 00000800:00000200:1.0:1790701497.325056:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701497.325060:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701497.325063:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bf7ee00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701497.325067:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701497.325113:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701497.325130:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bf7e900. 00000400:00000200:1.0:1790701497.325136:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701497.325140:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701497.325145:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b908 00000400:00000010:1.0:1790701497.325146:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701497.325454:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910bd1c00 x1877686323526528/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701497.325646:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701497.325649:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bf7ee00. 00000400:00000200:1.0:1790701497.325652:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701497.325656:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701497.325659:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bdd0 00000400:00000010:1.0:1790701497.325678:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bdd0. 00000100:00000200:1.0:1790701497.325685:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910bd2300 x1877686323526400/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701497.338984:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.338994:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701497.339004:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.339008:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.339014:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.339018:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503b00 00000400:00000200:1.0:1790701497.339024:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.339030:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x761 [1] + 224 00000400:00000200:1.0:1790701497.339043:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701497.339047:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701497.339060:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910bd2300 x1877686323526400/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701513 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701497.339092:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.339095:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701497.339100:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.339102:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.339105:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.339107:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503b80 00000400:00000200:1.0:1790701497.339110:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701497.339131:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x751 [1] + 224 00000400:00000200:1.0:1790701497.339145:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701497.339148:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701497.339453:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910bd1c00 x1877686323526528/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701513 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701497.339487:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69580. 00000400:00000200:1.0:1790701497.339492:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bee0 00000400:00000010:1.0:1790701497.339507:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bee0. 00000100:00000200:1.0:1790701497.339514:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910bd2300 x1877686323526400/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701497.339521:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910bd2300 x1877686323526400/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701513 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701497.339545:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69318. 00000400:00000200:1.0:1790701497.339547:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b220 00000400:00000010:1.0:1790701497.339549:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b220. 00000100:00000200:1.0:1790701497.339552:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910bd1c00 x1877686323526528/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701497.339559:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910bd1c00 x1877686323526528/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701513 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701497.339575:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c93bc23c00. 02000000:00000010:1.0:1790701497.339581:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bf7e100. 00000100:00000010:1.0:1790701497.339585:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd2300. 02000000:00000010:1.0:1790701497.339589:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c93bc22e00. 02000000:00000010:1.0:1790701497.339592:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bf7e300. 00000100:00000010:1.0:1790701497.339593:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd1c00. 00000100:00000010:1.0:1790701498.284363:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac360. 00000100:00000010:0.0:1790701498.285751:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac8a0. 00000001:02000400:1.0:1790701500.091244:0:134914:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 611: lnet_legacy2yaml: invalid hopcount ========================================================== 13:04:58 (1790701498) 00000100:00000010:0.0:1790701502.446215:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a21180. 02000000:00000010:0.0:1790701502.446237:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be79300. 00000100:00000010:0.0:1790701502.446269:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a20700. 02000000:00000010:0.0:1790701502.446274:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be79200. 02000000:00000010:0.0:1790701502.446305:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c94217ee00. 00000400:00000200:0.0:1790701502.446316:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701502.446321:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2bb0. 00000400:00000010:0.0:1790701502.446329:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a660. 00000100:00000200:0.0:1790701502.446336:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323526656, portal 25 00000400:00000010:0.0:1790701502.446341:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a7f8. 00000100:00000200:0.0:1790701502.446344:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323526656, offset 0 00000400:00000200:0.0:1790701502.446351:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701502.446359:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701502.446366:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701502.446371:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701502.446375:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701502.446379:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701502.446382:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701502.446385:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [124:124:256:4294967295] 192.168.206.116@tcp LPNI seq info [124:124:8:4294967295] 00000400:00000200:0.0:1790701502.446389:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701502.446398:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701502.446404:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701502.446414:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be79e00. 00000800:00000200:0.0:1790701502.446421:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701502.446429:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701502.446434:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be79e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701502.446449:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:0.0:1790701502.446453:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92249ef60. 00000100:00000010:0.0:1790701502.446464:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92249eba0. 02000000:00000010:0.0:1790701502.446484:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c94217e000. 00000400:00000200:0.0:1790701502.446489:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701502.446491:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2478. 00000400:00000010:0.0:1790701502.447292:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45acc0. 00000100:00000200:0.0:1790701502.447302:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323526784, portal 10 00000400:00000010:0.0:1790701502.447309:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701502.447312:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323526784, offset 0 00000400:00000200:0.0:1790701502.447319:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701502.447325:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701502.447330:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701502.447333:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701502.447336:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701502.447338:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701502.447340:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701502.447343:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [125:125:255:4294967295] 192.168.206.116@tcp LPNI seq info [125:125:7:4294967295] 00000400:00000200:0.0:1790701502.447347:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701502.447354:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701502.447358:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701502.447363:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be79b00. 00000800:00000200:0.0:1790701502.447368:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701502.447374:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701502.447377:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be79b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701502.447384:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701502.447430:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701502.447434:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be79e00. 00000400:00000200:0.0:1790701502.447439:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701502.447444:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701502.447449:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a7f8 00000400:00000010:0.0:1790701502.447451:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a7f8. 00000100:00000200:0.0:1790701502.447460:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a21180 x1877686323526656/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701502.448797:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701502.448804:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be79b00. 00000400:00000200:0.0:1790701502.448810:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701502.448816:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701502.448822:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a770 00000400:00000010:0.0:1790701502.448823:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701502.448831:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a20700 x1877686323526784/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701502.454445:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701502.454455:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701502.454463:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701502.454467:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701502.454473:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701502.454477:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503c00 00000400:00000200:1.0:1790701502.454482:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701502.454488:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x771 [1] + 224 00000400:00000200:1.0:1790701502.454498:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701502.454509:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701502.454523:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a21180 x1877686323526656/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701518 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701502.454553:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701502.454556:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701502.454561:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701502.454564:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701502.454567:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701502.454569:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503c80 00000400:00000200:1.0:1790701502.454573:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701502.454577:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x781 [1] + 224 00000400:00000200:1.0:1790701502.454587:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701502.454590:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701502.454596:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a20700 x1877686323526784/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701518 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701502.454626:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2478. 00000400:00000200:1.0:1790701502.454630:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45acc0 00000400:00000010:1.0:1790701502.454632:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45acc0. 00000100:00000200:1.0:1790701502.454638:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a20700 x1877686323526784/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701518 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701502.454647:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a20700 x1877686323526784/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701518 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701502.454676:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2bb0. 00000400:00000200:1.0:1790701502.454679:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a660 00000400:00000010:1.0:1790701502.454681:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a660. 00000100:00000200:1.0:1790701502.454685:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a21180 x1877686323526656/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701518 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701502.454693:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a21180 x1877686323526656/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701518 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701502.454734:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c94217e000. 02000000:00000010:1.0:1790701502.454740:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be79200. 00000100:00000010:1.0:1790701502.454745:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a20700. 02000000:00000010:1.0:1790701502.454749:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c94217ee00. 02000000:00000010:1.0:1790701502.454752:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be79300. 00000100:00000010:1.0:1790701502.454754:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a21180. 00000100:00000010:0.0:1790701503.404291:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92249ef60. 00000100:00000010:1.0:1790701503.408849:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92249eba0. 00000001:02000400:2.0:1790701507.050372:0:135485:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 612: lnet_legacy2yaml: malformed IP pattern ========================================================== 13:05:05 (1790701505) 00000100:00000010:0.0:1790701507.564257:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a23b80. 02000000:00000010:0.0:1790701507.564280:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be79400. 00000100:00000010:0.0:1790701507.564321:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a20a80. 02000000:00000010:0.0:1790701507.564325:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be79000. 02000000:00000010:0.0:1790701507.564355:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf6c00. 00000400:00000200:0.0:1790701507.564365:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701507.564370:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2bb0. 00000400:00000010:0.0:1790701507.564377:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701507.564382:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323527040, portal 10 00000400:00000010:0.0:1790701507.564389:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a7f8. 00000100:00000200:0.0:1790701507.564392:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323527040, offset 0 00000400:00000200:0.0:1790701507.564401:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701507.564408:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701507.564416:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701507.564421:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701507.564433:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701507.564437:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701507.564439:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701507.564443:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [126:126:256:4294967295] 192.168.206.116@tcp LPNI seq info [126:126:8:4294967295] 00000400:00000200:0.0:1790701507.564448:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701507.564457:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701507.564463:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701507.564467:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be79d00. 00000800:00000200:0.0:1790701507.564473:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701507.564480:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701507.564484:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be79d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701507.564497:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:0.0:1790701507.564502:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92249ea80. 00000100:00000010:0.0:1790701507.564513:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92249ee40. 02000000:00000010:0.0:1790701507.564534:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf6a00. 00000400:00000200:0.0:1790701507.564538:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701507.564540:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2478. 00000400:00000010:0.0:1790701507.564544:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a088. 00000100:00000200:0.0:1790701507.564547:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323526912, portal 25 00000400:00000010:0.0:1790701507.564550:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a5d8. 00000100:00000200:0.0:1790701507.564552:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323526912, offset 0 00000400:00000200:0.0:1790701507.564556:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701507.564560:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701507.564563:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701507.564566:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701507.564568:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701507.564570:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701507.564573:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701507.564576:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [127:127:255:4294967295] 192.168.206.116@tcp LPNI seq info [127:127:7:4294967295] 00000400:00000200:0.0:1790701507.564579:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701507.564585:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701507.564595:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701507.564598:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be79700. 00000800:00000200:0.0:1790701507.564602:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701507.564606:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701507.564609:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be79700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701507.564613:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701507.564655:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701507.564658:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be79d00. 00000400:00000200:0.0:1790701507.564664:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701507.564669:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701507.564673:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a7f8 00000400:00000010:0.0:1790701507.564675:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a7f8. 00000100:00000200:0.0:1790701507.564695:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a20a80 x1877686323527040/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701507.564759:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701507.564762:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be79700. 00000400:00000200:0.0:1790701507.564765:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701507.564768:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701507.564771:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a5d8 00000400:00000010:0.0:1790701507.564773:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a5d8. 00000100:00000200:0.0:1790701507.564777:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a23b80 x1877686323526912/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701507.569866:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701507.569877:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701507.569886:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701507.569891:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701507.569896:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701507.569900:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503d80 00000400:00000200:1.0:1790701507.569905:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701507.569911:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x791 [1] + 224 00000400:00000200:1.0:1790701507.570003:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701507.570009:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701507.570021:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a20a80 x1877686323527040/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701523 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701507.570115:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2bb0. 00000400:00000200:0.0:1790701507.570120:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a770 00000400:00000010:0.0:1790701507.570123:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701507.570129:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a20a80 x1877686323527040/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701523 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701507.570139:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a20a80 x1877686323527040/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701523 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701507.570470:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf6c00. 02000000:00000010:0.0:1790701507.570485:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be79000. 00000100:00000010:0.0:1790701507.570489:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a20a80. 00000400:00000200:0.0:1790701507.575577:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701507.575586:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701507.575595:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701507.575600:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701507.575606:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701507.575610:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503d00 00000400:00000200:0.0:1790701507.575615:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701507.575621:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x7a1 [1] + 224 00000400:00000200:0.0:1790701507.575636:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701507.575641:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701507.575656:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a23b80 x1877686323526912/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701523 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701507.575739:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2478. 00000400:00000200:1.0:1790701507.575745:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a088 00000400:00000010:1.0:1790701507.575748:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a088. 00000100:00000200:1.0:1790701507.575756:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a23b80 x1877686323526912/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701523 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701507.575767:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a23b80 x1877686323526912/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701523 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701507.575794:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf6a00. 02000000:00000010:1.0:1790701507.575799:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be79400. 00000100:00000010:1.0:1790701507.575803:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a23b80. 00000100:00000010:0.0:1790701508.525197:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92249ea80. 00000100:00000010:1.0:1790701508.531569:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92249ee40. 00000100:00000010:0.0:1790701512.699484:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a20a80. 02000000:00000010:0.0:1790701512.699505:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be79d00. 00000100:00000010:0.0:1790701512.699535:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a22a00. 02000000:00000010:0.0:1790701512.699540:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be79500. 02000000:00000010:0.0:1790701512.699586:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910fbe200. 00000400:00000200:0.0:1790701512.699597:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701512.699602:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2bb0. 00000400:00000010:0.0:1790701512.699609:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701512.699616:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323527296, portal 10 00000400:00000010:0.0:1790701512.699625:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a5d8. 00000100:00000200:0.0:1790701512.699628:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323527296, offset 0 00000400:00000200:0.0:1790701512.699638:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701512.699646:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.699666:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701512.699672:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701512.699677:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701512.699689:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701512.699692:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701512.699696:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [128:128:256:4294967295] 192.168.206.116@tcp LPNI seq info [128:128:8:4294967295] 00000400:00000200:0.0:1790701512.699702:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.699723:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701512.699731:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701512.699744:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be79000. 00000800:00000200:0.0:1790701512.699752:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701512.699759:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701512.699764:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be79000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701512.699780:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:0.0:1790701512.699786:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92249ecc0. 00000100:00000010:0.0:1790701512.699798:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92249ea80. 02000000:00000010:0.0:1790701512.699825:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910fbf800. 00000400:00000200:0.0:1790701512.699830:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701512.699834:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2898. 00000400:00000010:0.0:1790701512.699838:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a7f8. 00000100:00000200:0.0:1790701512.699841:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323527168, portal 25 00000400:00000010:0.0:1790701512.699845:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45aa18. 00000100:00000200:0.0:1790701512.699847:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323527168, offset 0 00000400:00000200:0.0:1790701512.699851:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701512.699855:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.699858:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701512.699861:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701512.699864:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701512.699866:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701512.699868:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701512.699870:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [129:129:255:4294967295] 192.168.206.116@tcp LPNI seq info [129:129:7:4294967295] 00000400:00000200:0.0:1790701512.699874:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.700057:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701512.700071:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701512.700075:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be79100. 00000800:00000200:0.0:1790701512.700079:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701512.700083:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701512.700086:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be79100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701512.700090:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701512.700136:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701512.700140:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be79000. 00000400:00000200:0.0:1790701512.700144:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701512.700149:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701512.700188:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a5d8 00000400:00000010:0.0:1790701512.700192:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a5d8. 00000100:00000200:0.0:1790701512.700201:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a22a00 x1877686323527296/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701512.700270:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701512.700274:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be79100. 00000400:00000200:0.0:1790701512.700278:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701512.700281:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701512.700284:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45aa18 00000400:00000010:0.0:1790701512.700286:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45aa18. 00000100:00000200:0.0:1790701512.700474:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a20a80 x1877686323527168/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701512.706582:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.706592:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701512.706601:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.706606:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.706611:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.706616:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503e80 00000400:00000200:0.0:1790701512.706621:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.706628:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x7b1 [1] + 224 00000400:00000200:0.0:1790701512.706649:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701512.706664:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701512.706678:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a22a00 x1877686323527296/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701528 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701512.706707:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.706713:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701512.706718:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.706720:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.706724:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.706727:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503e00 00000400:00000200:0.0:1790701512.706732:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701512.706747:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x7c1 [1] + 224 00000400:00000200:0.0:1790701512.706758:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701512.706762:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701512.706769:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a20a80 x1877686323527168/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701528 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701512.706800:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2898. 00000400:00000200:0.0:1790701512.706806:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a7f8 00000400:00000010:0.0:1790701512.706809:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a7f8. 00000100:00000200:0.0:1790701512.706815:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a20a80 x1877686323527168/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701528 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701512.706824:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a20a80 x1877686323527168/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701528 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701512.706846:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2bb0. 00000400:00000200:0.0:1790701512.706849:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a770 00000400:00000010:0.0:1790701512.706851:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701512.706855:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a22a00 x1877686323527296/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701528 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701512.706863:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a22a00 x1877686323527296/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701528 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701512.706890:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910fbf800. 02000000:00000010:0.0:1790701512.706896:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be79d00. 00000100:00000010:0.0:1790701512.706900:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a20a80. 02000000:00000010:0.0:1790701512.706904:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910fbe200. 02000000:00000010:0.0:1790701512.706906:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be79500. 00000100:00000010:0.0:1790701512.706908:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a22a00. 00000100:00000010:0.0:1790701513.645451:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92249ecc0. 00000100:00000010:1.0:1790701513.646440:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92249ea80. 00000001:02000400:0.0:1790701513.803023:0:136056:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 613: lnet_legacy2yaml: multiple IP patterns in single rule ========================================================== 13:05:12 (1790701512) 00000100:00000010:3.0:1790701517.806673:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908dae300. 02000000:00000010:3.0:1790701517.806692:0:8:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d57fb00. 00000100:00000010:3.0:1790701517.806719:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908daf100. 02000000:00000010:3.0:1790701517.806723:0:8:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d57f400. 02000000:00000010:3.0:1790701517.806754:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c924f75e00. 00000400:00000200:3.0:1790701517.806764:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701517.806768:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701517.806775:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35b28. 00000100:00000200:3.0:1790701517.806782:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323527424, portal 25 00000400:00000010:3.0:1790701517.806786:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d356e8. 00000100:00000200:3.0:1790701517.806790:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323527424, offset 0 00000400:00000200:3.0:1790701517.806799:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701517.806806:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701517.806820:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701517.806825:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701517.806829:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701517.806832:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701517.806835:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701517.806838:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [130:130:256:4294967295] 192.168.206.116@tcp LPNI seq info [130:130:8:4294967295] 00000400:00000200:3.0:1790701517.806843:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701517.806852:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701517.806859:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701517.806864:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f900. 00000800:00000200:3.0:1790701517.806870:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701517.806877:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701517.806881:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701517.806898:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701517.806902:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3720. 00000100:00000010:3.0:1790701517.806912:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3900. 02000000:00000010:3.0:1790701517.806932:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c923097600. 00000400:00000200:3.0:1790701517.806936:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701517.806939:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9f20. 00000400:00000010:3.0:1790701517.806942:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35220. 00000100:00000200:3.0:1790701517.806945:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323527552, portal 10 00000400:00000010:3.0:1790701517.806950:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35908. 00000100:00000200:3.0:1790701517.806952:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323527552, offset 0 00000400:00000200:3.0:1790701517.806956:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701517.806960:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701517.806963:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701517.806965:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701517.806967:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701517.806970:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701517.806972:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701517.806975:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [131:131:255:4294967295] 192.168.206.116@tcp LPNI seq info [131:131:7:4294967295] 00000800:00000200:0.0:1790701517.806983:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:3.0:1790701517.806984:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000010:0.0:1790701517.806987:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f900. 00000400:00000200:3.0:1790701517.806991:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000400:00000200:0.0:1790701517.806991:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1790701517.806996:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701517.806996:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000800:00000010:3.0:1790701517.806999:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f000. 00000400:00000200:0.0:1790701517.807002:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d356e8 00000800:00000200:3.0:1790701517.807003:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000400:00000010:0.0:1790701517.807004:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d356e8. 00000800:00000200:3.0:1790701517.807007:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701517.807009:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f000 type 1, nob 320 niov 1 nkiov 1 00000100:00000200:0.0:1790701517.807011:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908dae300 x1877686323527424/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701517.807015:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701517.807036:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701517.807038:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f000. 00000400:00000200:0.0:1790701517.807041:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701517.807044:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701517.807046:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35908 00000400:00000010:0.0:1790701517.807048:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35908. 00000100:00000200:0.0:1790701517.807052:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908daf100 x1877686323527552/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701517.809505:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701517.809514:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701517.809521:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701517.809526:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701517.809531:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701517.809535:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea503f00 00000400:00000200:1.0:1790701517.809541:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701517.809546:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x7d1 [1] + 224 00000400:00000200:1.0:1790701517.809560:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701517.809564:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701517.809577:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908dae300 x1877686323527424/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701533 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701517.809675:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:3.0:1790701517.809680:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35b28 00000400:00000010:3.0:1790701517.809682:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35b28. 00000100:00000200:3.0:1790701517.809689:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908dae300 x1877686323527424/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701533 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701517.809698:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908dae300 x1877686323527424/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701533 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701517.809723:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c924f75e00. 02000000:00000010:3.0:1790701517.809730:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d57fb00. 00000100:00000010:3.0:1790701517.809733:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908dae300. 00000400:00000200:1.0:1790701517.810284:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701517.810289:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701517.810294:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701517.810297:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701517.810301:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701517.810304:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea503f80 00000400:00000200:1.0:1790701517.810308:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701517.810313:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x7d9 [1] + 224 00000400:00000200:1.0:1790701517.810321:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701517.810325:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701517.810333:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908daf100 x1877686323527552/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701533 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701517.810500:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9f20. 00000400:00000200:2.0:1790701517.810505:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35220 00000400:00000010:2.0:1790701517.810507:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35220. 00000100:00000200:2.0:1790701517.810513:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908daf100 x1877686323527552/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701533 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701517.810532:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908daf100 x1877686323527552/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701533 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701517.810555:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c923097600. 02000000:00000010:2.0:1790701517.810560:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d57f400. 00000100:00000010:2.0:1790701517.810564:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908daf100. 00000100:00000010:0.0:1790701518.765229:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3720. 00000100:00000010:1.0:1790701518.766752:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3900. 00000001:02000400:0.0:1790701520.681377:0:136630:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 614: lnet_legacy2yaml: interfaces enumeration with multiple IP ranges ========================================================== 13:05:19 (1790701519) 00000100:00000010:1.0:1790701522.924323:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd1c00. 02000000:00000010:1.0:1790701522.924345:0:8:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bf7e900. 00000100:00000010:1.0:1790701522.924387:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd2300. 02000000:00000010:1.0:1790701522.924408:0:8:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bf7e100. 02000000:00000010:1.0:1790701522.924443:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910fbfe00. 00000400:00000200:1.0:1790701522.924453:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701522.924457:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69318. 00000400:00000010:1.0:1790701522.924464:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b220. 00000100:00000200:1.0:1790701522.924470:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323527808, portal 10 00000400:00000010:1.0:1790701522.924477:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bee0. 00000100:00000200:1.0:1790701522.924480:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323527808, offset 0 00000400:00000200:1.0:1790701522.924488:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701522.924495:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.924502:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701522.924507:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701522.924511:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701522.924515:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701522.924518:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701522.924521:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [132:132:256:4294967295] 192.168.206.116@tcp LPNI seq info [132:132:8:4294967295] 00000400:00000200:1.0:1790701522.924539:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.924549:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701522.924555:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701522.924559:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bf7ee00. 00000800:00000200:1.0:1790701522.924565:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701522.924572:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701522.924577:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bf7ee00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701522.924591:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:1.0:1790701522.924597:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac2a0. 00000100:00000010:1.0:1790701522.924621:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cacc60. 02000000:00000010:1.0:1790701522.924642:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910fbf600. 00000400:00000200:1.0:1790701522.924646:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701522.924648:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69580. 00000400:00000010:1.0:1790701522.924652:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bdd0. 00000100:00000200:1.0:1790701522.924655:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323527680, portal 25 00000400:00000010:1.0:1790701522.924665:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701522.924668:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323527680, offset 0 00000400:00000200:1.0:1790701522.924672:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701522.924676:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.924680:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701522.924682:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701522.924684:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701522.924687:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701522.924689:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701522.924691:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [133:133:255:4294967295] 192.168.206.116@tcp LPNI seq info [133:133:7:4294967295] 00000400:00000200:1.0:1790701522.924695:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.924700:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701522.924705:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701522.924708:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bf7e600. 00000800:00000200:1.0:1790701522.924711:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701522.924715:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701522.924717:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bf7e600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701522.924721:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701522.924768:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701522.924772:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bf7ee00. 00000400:00000200:1.0:1790701522.924778:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701522.924782:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701522.924788:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bee0 00000400:00000010:1.0:1790701522.924790:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bee0. 00000100:00000200:1.0:1790701522.924799:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910bd2300 x1877686323527808/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701522.924872:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701522.924875:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bf7e600. 00000400:00000200:1.0:1790701522.924878:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701522.924881:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701522.924884:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b908 00000400:00000010:1.0:1790701522.924891:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701522.924897:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910bd1c00 x1877686323527680/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701522.927409:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.927417:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701522.927423:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.927428:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.927433:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.927437:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea504000 00000400:00000200:1.0:1790701522.927442:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.927447:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x7f1 [1] + 224 00000400:00000200:1.0:1790701522.927456:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701522.927460:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701522.927471:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910bd1c00 x1877686323527680/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701538 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701522.927501:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.927504:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701522.927508:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.927510:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.927513:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.927515:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea504080 00000400:00000200:1.0:1790701522.927518:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701522.927522:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x7e1 [1] + 224 00000400:00000200:1.0:1790701522.927651:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701522.927655:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000010:0.0:1790701522.927839:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69580. 00000400:00000200:0.0:1790701522.927845:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bdd0 00000400:00000010:0.0:1790701522.927858:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bdd0. 00000100:00000200:0.0:1790701522.927865:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910bd1c00 x1877686323527680/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701538 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701522.927875:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910bd1c00 x1877686323527680/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701538 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701522.927901:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910fbf600. 02000000:00000010:0.0:1790701522.927905:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bf7e900. 00000100:00000010:0.0:1790701522.927909:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd1c00. 00000100:00000200:1.0:1790701522.928044:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910bd2300 x1877686323527808/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701538 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701522.928335:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69318. 00000400:00000200:1.0:1790701522.928339:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b220 00000400:00000010:1.0:1790701522.928347:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b220. 00000100:00000200:1.0:1790701522.928352:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910bd2300 x1877686323527808/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701538 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701522.928360:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910bd2300 x1877686323527808/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701538 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701522.928379:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910fbfe00. 02000000:00000010:1.0:1790701522.928382:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bf7e100. 00000100:00000010:1.0:1790701522.928385:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd2300. 00000100:00000010:1.0:1790701523.884237:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cacc60. 00000100:00000010:0.0:1790701523.890890:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac2a0. 00000001:02000400:0.0:1790701527.950557:0:137202:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 615: lnet_legacy2yaml: modprobe configuration file input ========================================================== 13:05:26 (1790701526) 00000100:00000010:1.0:1790701528.047849:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd2300. 02000000:00000010:1.0:1790701528.047888:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bf7e100. 02000000:00000010:1.0:1790701528.047950:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91885d200. 00000400:00000200:1.0:1790701528.047963:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701528.047967:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69318. 00000400:00000010:1.0:1790701528.047977:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b220. 00000100:00000200:1.0:1790701528.047987:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323527936, portal 25 00000400:00000010:1.0:1790701528.047994:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701528.047997:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323527936, offset 0 00000400:00000200:1.0:1790701528.048008:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701528.048017:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.048027:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701528.048034:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701528.048039:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701528.048043:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701528.048046:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701528.048050:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [134:134:256:4294967295] 192.168.206.116@tcp LPNI seq info [134:134:8:4294967295] 00000400:00000200:1.0:1790701528.048055:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.048065:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701528.048073:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701528.048078:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bf7e600. 00000800:00000200:1.0:1790701528.048098:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701528.048107:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701528.048112:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bf7e600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701528.048128:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:1.0:1790701528.048135:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac660. 00000100:00000010:1.0:1790701528.048184:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cacc60. 00000800:00000200:1.0:1790701528.048329:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701528.048334:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bf7e600. 00000400:00000200:1.0:1790701528.048340:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701528.048345:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701528.048352:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b908 00000400:00000010:1.0:1790701528.048355:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701528.048366:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910bd2300 x1877686323527936/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1790701528.048394:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd2680. 02000000:00000010:1.0:1790701528.048402:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bf7e600. 02000000:00000010:1.0:1790701528.048429:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91885d800. 00000400:00000200:1.0:1790701528.048434:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701528.048437:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69370. 00000400:00000010:1.0:1790701528.048441:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701528.048445:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323528064, portal 10 00000400:00000010:1.0:1790701528.048451:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bee0. 00000100:00000200:1.0:1790701528.048454:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323528064, offset 0 00000400:00000200:1.0:1790701528.048458:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701528.048479:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.048483:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701528.048487:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701528.048490:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701528.048492:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701528.048495:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701528.048497:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [135:135:256:4294967295] 192.168.206.116@tcp LPNI seq info [135:135:8:4294967295] 00000400:00000200:1.0:1790701528.048501:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.048514:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701528.048519:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701528.048523:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bf7ee00. 00000800:00000200:1.0:1790701528.048527:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701528.048532:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701528.048535:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bf7ee00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701528.048547:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701528.048572:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701528.048589:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bf7ee00. 00000400:00000200:1.0:1790701528.048593:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701528.048596:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701528.048600:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bee0 00000400:00000010:1.0:1790701528.048602:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bee0. 00000100:00000200:1.0:1790701528.048620:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910bd2680 x1877686323528064/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701528.052792:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.052806:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701528.052817:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.052824:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.052830:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.052835:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea504180 00000400:00000200:1.0:1790701528.052842:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.052849:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x811 [1] + 224 00000400:00000200:1.0:1790701528.052934:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701528.052940:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701528.052958:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910bd2680 x1877686323528064/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701544 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701528.053014:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69370. 00000400:00000200:1.0:1790701528.053019:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b908 00000400:00000010:1.0:1790701528.053022:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b908. 00000100:00000200:1.0:1790701528.053047:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910bd2680 x1877686323528064/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701544 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701528.053057:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910bd2680 x1877686323528064/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701544 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701528.053099:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91885d800. 02000000:00000010:1.0:1790701528.053107:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bf7e600. 00000100:00000010:1.0:1790701528.053112:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd2680. 00000400:00000200:1.0:1790701528.055221:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.055234:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701528.055245:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.055251:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.055257:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.055262:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea504100 00000400:00000200:1.0:1790701528.055268:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701528.055275:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x801 [1] + 224 00000400:00000200:1.0:1790701528.055293:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701528.055298:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701528.055314:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910bd2300 x1877686323527936/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701544 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701528.055366:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69318. 00000400:00000200:1.0:1790701528.055371:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b220 00000400:00000010:1.0:1790701528.055374:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b220. 00000100:00000200:1.0:1790701528.055380:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910bd2300 x1877686323527936/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701544 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701528.055390:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910bd2300 x1877686323527936/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701544 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701528.055420:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91885d200. 02000000:00000010:1.0:1790701528.055427:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bf7e100. 00000100:00000010:1.0:1790701528.055432:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd2300. 00000100:00000010:1.0:1790701529.004267:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cacc60. 00000100:00000010:0.0:1790701529.006785:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac660. 00000100:00000010:2.0:1790701533.164307:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95f80. 02000000:00000010:2.0:1790701533.164339:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be7fc00. 00000100:00000010:2.0:1790701533.164378:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94a80. 02000000:00000010:2.0:1790701533.164383:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be7f100. 02000000:00000010:2.0:1790701533.164427:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c9183ef600. 00000400:00000200:2.0:1790701533.164439:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701533.164444:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823c60. 00000400:00000010:2.0:1790701533.164453:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeebb0. 00000100:00000200:2.0:1790701533.164462:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323528192, portal 25 00000400:00000010:2.0:1790701533.164468:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee990. 00000100:00000200:2.0:1790701533.164472:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323528192, offset 0 00000400:00000200:2.0:1790701533.164496:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701533.164506:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701533.164516:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701533.164522:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701533.164527:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701533.164531:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701533.164535:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701533.164539:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [136:136:256:4294967295] 192.168.206.116@tcp LPNI seq info [136:136:8:4294967295] 00000400:00000200:2.0:1790701533.164544:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701533.164564:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701533.164571:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701533.164578:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be7f600. 00000800:00000200:2.0:1790701533.164586:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701533.164605:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701533.164610:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be7f600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701533.164625:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:2.0:1790701533.164630:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305c840. 00000100:00000010:2.0:1790701533.164643:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c92305c5a0. 02000000:00000010:2.0:1790701533.164665:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c9183eee00. 00000400:00000200:2.0:1790701533.164669:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701533.164672:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c9188235d8. 00000400:00000010:2.0:1790701533.165916:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee198. 00000100:00000200:2.0:1790701533.165935:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323528320, portal 10 00000400:00000010:2.0:1790701533.165949:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee000. 00000100:00000200:2.0:1790701533.165953:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323528320, offset 0 00000400:00000200:2.0:1790701533.165963:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701533.165972:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701533.165979:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701533.165984:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701533.165987:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701533.165991:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701533.165993:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701533.165997:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [137:137:255:4294967295] 192.168.206.116@tcp LPNI seq info [137:137:7:4294967295] 00000400:00000200:2.0:1790701533.166001:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701533.166009:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701533.166015:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701533.166020:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be7f500. 00000800:00000200:2.0:1790701533.166026:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701533.166034:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701533.166039:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be7f500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701533.166047:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701533.167564:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701533.167574:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be7f600. 00000400:00000200:1.0:1790701533.167582:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701533.167593:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701533.167602:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee990 00000400:00000010:1.0:1790701533.167605:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee990. 00000100:00000200:1.0:1790701533.167619:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b95f80 x1877686323528192/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701533.167705:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701533.167708:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be7f500. 00000400:00000200:1.0:1790701533.167712:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701533.167716:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701533.167719:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee000 00000400:00000010:1.0:1790701533.167721:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee000. 00000100:00000200:1.0:1790701533.167727:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b94a80 x1877686323528320/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701533.190952:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701533.190966:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701533.190977:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701533.190983:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701533.190989:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701533.190994:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea504280 00000400:00000200:0.0:1790701533.191001:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701533.191008:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x829 [1] + 224 00000400:00000200:0.0:1790701533.191023:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701533.191028:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701533.191045:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b94a80 x1877686323528320/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701549 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701533.191094:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701533.191098:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701533.191102:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701533.191105:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701533.191107:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701533.191110:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea504200 00000400:00000200:0.0:1790701533.191122:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701533.191127:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x821 [1] + 224 00000400:00000200:0.0:1790701533.191142:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701533.191146:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701533.191168:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b95f80 x1877686323528192/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701549 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701533.191808:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c9188235d8. 00000400:00000200:2.0:1790701533.191815:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee198 00000400:00000010:2.0:1790701533.191818:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee198. 00000100:00000200:2.0:1790701533.191835:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b94a80 x1877686323528320/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701533.191845:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b94a80 x1877686323528320/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701549 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701533.191881:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823c60. 00000400:00000200:2.0:1790701533.191885:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeebb0 00000400:00000010:2.0:1790701533.191887:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeebb0. 00000100:00000200:2.0:1790701533.191891:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b95f80 x1877686323528192/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701533.191899:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b95f80 x1877686323528192/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701549 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701533.191916:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c9183eee00. 02000000:00000010:2.0:1790701533.191922:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be7f100. 00000100:00000010:2.0:1790701533.191927:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94a80. 02000000:00000010:2.0:1790701533.191932:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c9183ef600. 02000000:00000010:2.0:1790701533.191934:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be7fc00. 00000100:00000010:2.0:1790701533.191936:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95f80. 00000100:00000010:0.0:1790701534.127643:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305c840. 00000100:00000010:1.0:1790701534.131322:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c92305c5a0. 00000001:02000400:0.0:1790701537.015021:0:137776:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 616: lnet_legacy2yaml: combining modprobe file with explicit ip2nets file ========================================================== 13:05:35 (1790701535) 00000100:00000010:3.0:1790701538.284861:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908dae300. 02000000:00000010:3.0:1790701538.284882:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d57fb00. 00000100:00000010:3.0:1790701538.284917:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908dace00. 02000000:00000010:3.0:1790701538.284922:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d57ff00. 02000000:00000010:3.0:1790701538.284953:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c9183ee600. 00000400:00000200:3.0:1790701538.284962:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701538.284966:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701538.284973:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35b28. 00000100:00000200:3.0:1790701538.284980:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323528576, portal 10 00000400:00000010:3.0:1790701538.284987:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35c38. 00000100:00000200:3.0:1790701538.284990:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323528576, offset 0 00000400:00000200:3.0:1790701538.285000:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701538.285017:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701538.285024:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701538.285029:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701538.285033:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701538.285037:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701538.285039:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701538.285042:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [138:138:256:4294967295] 192.168.206.116@tcp LPNI seq info [138:138:8:4294967295] 00000400:00000200:3.0:1790701538.285047:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701538.285067:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701538.285073:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701538.285078:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f200. 00000800:00000200:3.0:1790701538.285084:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701538.285091:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701538.285095:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f200 type 1, nob 320 niov 1 nkiov 1 02000000:00000010:2.0:1790701538.285184:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910fbfe00. 00000400:00000200:2.0:1790701538.285190:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701538.285193:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823c60. 00000400:00000010:2.0:1790701538.285197:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeebb0. 00000100:00000200:2.0:1790701538.285201:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323528448, portal 25 00000400:00000010:2.0:1790701538.285204:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee198. 00000100:00000200:2.0:1790701538.285206:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323528448, offset 0 00000400:00000200:2.0:1790701538.285211:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701538.285214:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701538.285218:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701538.285220:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701538.285222:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701538.285224:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701538.285226:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701538.285234:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [139:139:255:4294967295] 192.168.206.116@tcp LPNI seq info [139:139:7:4294967295] 00000400:00000200:2.0:1790701538.285238:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701538.285244:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701538.285248:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701538.285252:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be7fc00. 00000800:00000200:2.0:1790701538.285255:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701538.285260:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701538.285262:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be7fc00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701538.285300:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701538.285307:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec36c0. 00000100:00000010:3.0:1790701538.285317:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3600. 00000400:00000200:2.0:1790701538.286343:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701538.286716:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701538.286721:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f200. 00000400:00000200:1.0:1790701538.286727:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701538.286733:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701538.286740:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35c38 00000400:00000010:1.0:1790701538.286742:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35c38. 00000100:00000200:1.0:1790701538.286752:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908dace00 x1877686323528576/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701538.286830:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701538.286833:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be7fc00. 00000400:00000200:1.0:1790701538.286837:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701538.286840:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701538.286843:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee198 00000400:00000010:1.0:1790701538.286845:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee198. 00000100:00000200:1.0:1790701538.286857:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908dae300 x1877686323528448/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701538.294470:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701538.294480:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701538.294489:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701538.294494:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701538.294499:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701538.294502:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea504380 00000400:00000200:0.0:1790701538.294509:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701538.294516:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x831 [1] + 224 00000400:00000200:0.0:1790701538.294543:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701538.294548:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701538.294561:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908dace00 x1877686323528576/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701554 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701538.294771:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:3.0:1790701538.294777:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35b28 00000400:00000010:3.0:1790701538.294779:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35b28. 00000100:00000200:3.0:1790701538.294785:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908dace00 x1877686323528576/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701554 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701538.294794:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908dace00 x1877686323528576/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701554 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701538.294824:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c9183ee600. 02000000:00000010:3.0:1790701538.294829:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d57ff00. 00000100:00000010:3.0:1790701538.294833:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908dace00. 00000400:00000200:0.0:1790701538.294974:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701538.294979:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701538.294984:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701538.294987:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701538.294990:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701538.294993:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea504300 00000400:00000200:0.0:1790701538.294997:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701538.295001:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x839 [1] + 224 00000400:00000200:0.0:1790701538.295017:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701538.295020:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701538.295027:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908dae300 x1877686323528448/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701554 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701538.295134:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823c60. 00000400:00000200:3.0:1790701538.295138:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeebb0 00000400:00000010:3.0:1790701538.295140:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeebb0. 00000100:00000200:3.0:1790701538.295255:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908dae300 x1877686323528448/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701554 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701538.295268:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908dae300 x1877686323528448/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701554 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701538.295283:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910fbfe00. 02000000:00000010:3.0:1790701538.295286:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d57fb00. 00000100:00000010:3.0:1790701538.295289:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908dae300. 00000100:00000010:1.0:1790701539.245718:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3600. 00000100:00000010:0.0:1790701539.245949:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec36c0. 00000100:00000010:2.0:1790701543.405906:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95f80. 02000000:00000010:2.0:1790701543.405928:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be7fd00. 02000000:00000010:3.0:1790701543.405996:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c9183ee000. 00000400:00000200:3.0:1790701543.406006:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701543.406010:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701543.406017:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35b28. 00000100:00000200:3.0:1790701543.406023:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323528704, portal 25 00000400:00000010:3.0:1790701543.406029:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d353b8. 00000100:00000200:3.0:1790701543.406033:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323528704, offset 0 00000400:00000200:3.0:1790701543.406041:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701543.406122:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701543.406131:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701543.406136:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701543.406140:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701543.406144:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701543.406147:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701543.406150:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [140:140:256:4294967295] 192.168.206.116@tcp LPNI seq info [140:140:8:4294967295] 00000400:00000200:3.0:1790701543.406175:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701543.406185:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701543.406192:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701543.406197:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f500. 00000800:00000200:3.0:1790701543.406204:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701543.406211:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701543.406216:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701543.406229:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701543.406239:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3ba0. 00000100:00000010:3.0:1790701543.406250:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c917ec3c60. 00000100:00000010:2.0:1790701543.406400:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94a80. 02000000:00000010:2.0:1790701543.406408:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be7f700. 00000800:00000200:0.0:1790701543.406413:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701543.406417:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f500. 00000400:00000200:0.0:1790701543.406422:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701543.406428:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701543.406434:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d353b8 00000400:00000010:0.0:1790701543.406436:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d353b8. 00000100:00000200:0.0:1790701543.406445:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b95f80 x1877686323528704/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701543.406448:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c9183ee800. 00000400:00000200:3.0:1790701543.406541:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701543.406547:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9a50. 00000400:00000010:3.0:1790701543.406552:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35110. 00000100:00000200:3.0:1790701543.406555:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323528832, portal 10 00000400:00000010:3.0:1790701543.406561:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35dd0. 00000100:00000200:3.0:1790701543.406563:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323528832, offset 0 00000400:00000200:3.0:1790701543.406568:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701543.406572:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701543.406576:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701543.406579:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701543.406581:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701543.406583:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701543.406585:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701543.406588:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [141:141:256:4294967295] 192.168.206.116@tcp LPNI seq info [141:141:8:4294967295] 00000400:00000200:3.0:1790701543.406591:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701543.406733:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701543.406738:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701543.406742:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d57f700. 00000800:00000200:3.0:1790701543.406746:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701543.406751:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701543.406753:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d57f700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701543.406773:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701543.407427:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701543.407433:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d57f700. 00000400:00000200:0.0:1790701543.407438:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701543.407443:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701543.407448:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35dd0 00000400:00000010:0.0:1790701543.407450:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35dd0. 00000100:00000200:0.0:1790701543.407456:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b94a80 x1877686323528832/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701543.411079:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701543.411090:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701543.411098:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701543.411103:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701543.411109:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701543.411113:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea504400 00000400:00000200:0.0:1790701543.411119:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701543.411124:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x841 [1] + 224 00000400:00000200:0.0:1790701543.411137:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701543.411148:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701543.411181:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b95f80 x1877686323528704/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701559 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701543.411216:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701543.411220:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701543.411224:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701543.411227:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701543.411229:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701543.411232:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea504480 00000400:00000200:0.0:1790701543.411235:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701543.411239:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x849 [1] + 224 00000400:00000200:0.0:1790701543.411248:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701543.411251:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701543.411257:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b94a80 x1877686323528832/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701559 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701543.411605:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9a50. 00000400:00000200:2.0:1790701543.411612:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35110 00000400:00000010:2.0:1790701543.411614:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35110. 00000100:00000200:2.0:1790701543.411621:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b94a80 x1877686323528832/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701559 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701543.411631:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b94a80 x1877686323528832/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701559 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701543.411731:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:2.0:1790701543.411735:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35b28 00000400:00000010:2.0:1790701543.411737:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35b28. 00000100:00000200:2.0:1790701543.411741:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b95f80 x1877686323528704/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701559 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701543.411748:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b95f80 x1877686323528704/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701559 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701543.411763:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c9183ee800. 02000000:00000010:2.0:1790701543.411767:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be7f700. 00000100:00000010:2.0:1790701543.411778:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94a80. 02000000:00000010:2.0:1790701543.411782:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c9183ee000. 02000000:00000010:2.0:1790701543.411784:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be7fd00. 00000100:00000010:2.0:1790701543.411786:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95f80. 00000100:00000010:0.0:1790701544.368245:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3ba0. 00000100:00000010:1.0:1790701544.372234:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c917ec3c60. 00000001:02000400:2.0:1790701545.724604:0:138353:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 617: lnet_legacy2yaml: round-trip tunables from LNet to modprobe to YAML ========================================================== 13:05:44 (1790701544) 00000001:02000400:3.0:1790701546.249417:0:138368:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure 00000001:02000400:0.0:1790701547.820109:0:138494:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_617: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240 00000100:00000010:2.0:1790701548.524237:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95f80. 02000000:00000010:2.0:1790701548.524264:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be7f300. 00000100:00000010:2.0:1790701548.524298:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94a80. 02000000:00000010:2.0:1790701548.524305:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93be7f200. 02000000:00000010:2.0:1790701548.524348:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c94217e200. 00000400:00000200:2.0:1790701548.524361:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701548.524367:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c9188235d8. 00000400:00000010:2.0:1790701548.524376:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee770. 00000100:00000200:2.0:1790701548.524386:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323528960, portal 25 00000400:00000010:2.0:1790701548.524392:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee7f8. 00000100:00000200:2.0:1790701548.524396:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323528960, offset 0 00000400:00000200:2.0:1790701548.524407:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701548.524416:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701548.524426:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701548.524432:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701548.524438:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701548.524442:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701548.524615:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701548.524620:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [142:142:256:4294967295] 192.168.206.116@tcp LPNI seq info [142:142:8:4294967295] 00000400:00000200:2.0:1790701548.524626:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701548.524637:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701548.524645:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701548.524651:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be7f900. 00000800:00000200:2.0:1790701548.524658:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701548.524667:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701548.524682:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be7f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701548.524707:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:2.0:1790701548.524714:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c907ca02a0. 00000100:00000010:2.0:1790701548.524727:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c907ca0ae0. 02000000:00000010:2.0:1790701548.524753:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c94217e000. 00000400:00000200:2.0:1790701548.524758:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701548.524761:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823160. 00000400:00000010:2.0:1790701548.524765:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee2a8. 00000100:00000200:2.0:1790701548.524768:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323529088, portal 10 00000400:00000010:2.0:1790701548.524774:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee330. 00000100:00000200:2.0:1790701548.524776:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323529088, offset 0 00000400:00000200:2.0:1790701548.524781:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701548.524785:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701548.524789:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701548.524792:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701548.524794:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701548.524796:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701548.524799:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701548.524802:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [143:143:255:4294967295] 192.168.206.116@tcp LPNI seq info [143:143:7:4294967295] 00000400:00000200:2.0:1790701548.524806:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701548.524812:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701548.524816:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701548.524819:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93be7fd00. 00000800:00000200:2.0:1790701548.524823:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701548.524827:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701548.524830:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93be7fd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701548.524836:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701548.525100:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701548.525105:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be7f900. 00000400:00000200:0.0:1790701548.525112:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701548.525119:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701548.525126:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee7f8 00000400:00000010:0.0:1790701548.525128:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee7f8. 00000100:00000200:0.0:1790701548.525139:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b95f80 x1877686323528960/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701548.525277:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701548.525281:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93be7fd00. 00000400:00000200:0.0:1790701548.525286:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701548.525292:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701548.525295:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee330 00000400:00000010:0.0:1790701548.525305:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee330. 00000100:00000200:0.0:1790701548.525313:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b94a80 x1877686323529088/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701548.529423:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701548.529437:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701548.529451:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701548.529458:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701548.529466:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701548.529470:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea504580 00000400:00000200:1.0:1790701548.529491:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701548.529499:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x859 [1] + 224 00000400:00000200:1.0:1790701548.529520:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701548.529526:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701548.529545:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b94a80 x1877686323529088/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701564 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701548.529654:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823160. 00000400:00000200:3.0:1790701548.529664:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee2a8 00000400:00000010:3.0:1790701548.529667:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee2a8. 00000100:00000200:3.0:1790701548.529931:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b94a80 x1877686323529088/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701564 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701548.529959:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b94a80 x1877686323529088/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701564 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701548.529994:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c94217e000. 02000000:00000010:3.0:1790701548.530002:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be7f200. 00000100:00000010:3.0:1790701548.530006:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94a80. 00000400:00000200:1.0:1790701548.531188:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701548.531199:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701548.531209:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701548.531215:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701548.531220:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701548.531224:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea504500 00000400:00000200:1.0:1790701548.531230:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701548.531236:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x851 [1] + 224 00000400:00000200:1.0:1790701548.531253:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701548.531257:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701548.531272:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b95f80 x1877686323528960/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701564 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701548.531365:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c9188235d8. 00000400:00000200:3.0:1790701548.531374:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee770 00000400:00000010:3.0:1790701548.531377:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee770. 00000100:00000200:3.0:1790701548.531388:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b95f80 x1877686323528960/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701564 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701548.531398:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b95f80 x1877686323528960/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701564 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701548.531422:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c94217e200. 02000000:00000010:3.0:1790701548.531427:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93be7f300. 00000100:00000010:3.0:1790701548.531431:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95f80. 00000100:00000010:0.0:1790701549.484713:0:106284:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c907ca02a0. 00000100:00000010:1.0:1790701549.491004:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c907ca0ae0. Debug log: 3096 lines, 3096 kept, 0 dropped, 0 bad.