00000100:00000010:1.0F:1790701348.845054:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd0700.
02000000:00000010:1.0:1790701348.845096:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c923075700.
02000000:00000010:1.0:1790701348.845141:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf6200.
00000400:00000200:1.0:1790701348.845151:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701348.845183:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e693c8.
00000400:00000010:1.0:1790701348.845189:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b110.
00000100:00000200:1.0:1790701348.845196:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323518976, portal 25
00000400:00000010:1.0:1790701348.845201:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bc38.
00000100:00000200:1.0:1790701348.845203:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323518976, offset 0
00000400:00000200:1.0:1790701348.845212:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701348.845219:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701348.845227:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701348.845232:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701348.845237:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701348.845240:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701348.845243:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701348.845247:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [64:64:256:4294967295] 192.168.206.116@tcp LPNI seq info [64:64:8:4294967295]
00000400:00000200:1.0:1790701348.845251:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701348.845260:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701348.845265:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701348.845270:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075300.
00000800:00000200:1.0:1790701348.845276:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701348.845284:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701348.845288:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701348.845301:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:1.0:1790701348.845307:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cacd80.
00000100:00000010:1.0:1790701348.845319:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac060.
00000800:00000200:1.0:1790701348.845418:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701348.845422:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075300.
00000400:00000200:1.0:1790701348.845427:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701348.845432:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701348.845436:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bc38
00000400:00000010:1.0:1790701348.845444:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bc38.
00000100:00000200:1.0:1790701348.845452:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910bd0700 x1877686323518976/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1790701348.845474:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd0380.
02000000:00000010:1.0:1790701348.845480:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c923075300.
02000000:00000010:1.0:1790701348.845510:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf7a00.
00000400:00000200:1.0:1790701348.845514:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701348.845517:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69f20.
00000400:00000010:1.0:1790701348.845520:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bc38.
00000100:00000200:1.0:1790701348.845523:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323519104, portal 10
00000400:00000010:1.0:1790701348.845528:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775ba18.
00000100:00000200:1.0:1790701348.845531:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323519104, offset 0
00000400:00000200:1.0:1790701348.845535:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701348.845539:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701348.845543:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701348.845546:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701348.845548:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701348.845550:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701348.845553:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701348.845556:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [65:65:256:4294967295] 192.168.206.116@tcp LPNI seq info [65:65:8:4294967295]
00000400:00000200:1.0:1790701348.845560:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701348.845566:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701348.845570:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701348.845573:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075a00.
00000800:00000200:1.0:1790701348.845576:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701348.845580:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701348.845583:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701348.845593:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:1.0:1790701348.845615:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701348.845618:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075a00.
00000400:00000200:1.0:1790701348.845620:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701348.845629:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701348.845632:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775ba18
00000400:00000010:1.0:1790701348.845634:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775ba18.
00000100:00000200:1.0:1790701348.845639:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910bd0380 x1877686323519104/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0F:1790701348.857192:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701348.857210:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701348.857218:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701348.857223:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701348.857228:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701348.857232:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea501e80
00000400:00000200:0.0:1790701348.857237:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701348.857243:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x4e1 [1] + 224
00000400:00000200:0.0:1790701348.857256:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701348.857261:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701348.857275:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910bd0380 x1877686323519104/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701364 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701348.857311:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701348.857315:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701348.857319:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701348.857322:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701348.857324:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701348.857327:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea501e00
00000400:00000200:0.0:1790701348.857330:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701348.857334:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x4d1 [1] + 224
00000400:00000200:0.0:1790701348.857344:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701348.857347:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701348.857353:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910bd0700 x1877686323518976/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701364 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1790701348.857390:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e693c8.
00000400:00000200:0.0:1790701348.857395:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b110
00000400:00000010:0.0:1790701348.857397:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b110.
00000100:00000200:0.0:1790701348.857403:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910bd0700 x1877686323518976/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701364 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1790701348.857412:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910bd0700 x1877686323518976/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701364 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1790701348.857600:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf6200.
02000000:00000010:0.0:1790701348.857607:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c923075700.
00000100:00000010:0.0:1790701348.857611:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd0700.
00000400:00000010:1.0:1790701348.857689:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69f20.
00000400:00000200:1.0:1790701348.857693:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bc38
00000400:00000010:1.0:1790701348.857695:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bc38.
00000100:00000200:1.0:1790701348.857701:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910bd0380 x1877686323519104/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701364 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1790701348.857708:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910bd0380 x1877686323519104/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701364 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1790701348.857729:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf7a00.
02000000:00000010:1.0:1790701348.857733:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c923075300.
00000100:00000010:1.0:1790701348.857735:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd0380.
00000100:00000010:1.0:1790701349.804260:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac060.
00000100:00000010:0.0:1790701349.804411:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cacd80.
00000100:00000010:1.0:1790701353.964293:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd0380.
02000000:00000010:1.0:1790701353.964315:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c923075300.
00000100:00000010:1.0:1790701353.964359:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd3b80.
02000000:00000010:1.0:1790701353.964364:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c923075a00.
02000000:00000010:1.0:1790701353.964404:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf7800.
00000400:00000200:1.0:1790701353.964414:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701353.964418:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69f20.
00000400:00000010:1.0:1790701353.964424:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bc38.
00000100:00000200:1.0:1790701353.964430:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323519360, portal 10
00000400:00000010:1.0:1790701353.964437:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775ba18.
00000100:00000200:1.0:1790701353.964439:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323519360, offset 0
00000400:00000200:1.0:1790701353.964448:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701353.964455:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701353.964462:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701353.964467:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000010:0.0:1790701353.964579:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a04e600.
00000400:00000200:0.0:1790701353.964585:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:0.0:1790701353.964587:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2898.
00000400:00000010:0.0:1790701353.964591:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a4c8.
00000100:00000200:0.0:1790701353.964595:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323519232, portal 25
00000400:00000010:0.0:1790701353.964598:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a440.
00000100:00000200:0.0:1790701353.964600:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323519232, offset 0
00000400:00000200:0.0:1790701353.964605:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701353.964639:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701353.964642:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701353.964645:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701353.964648:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [66:66:256:4294967295] 192.168.206.116@tcp LPNI seq info [66:66:8:4294967295]
00000400:00000200:1.0:1790701353.964653:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701353.964663:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701353.964669:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701353.964673:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075c00.
00000800:00000200:1.0:1790701353.964680:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701353.964687:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701353.964691:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075c00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701353.964706:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:1.0:1790701353.964711:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac4e0.
00000100:00000010:1.0:1790701353.964722:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac0c0.
00000800:00000200:1.0:1790701353.964827:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701353.964831:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075c00.
00000400:00000200:1.0:1790701353.964836:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701353.964841:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701353.973052:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701353.973065:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:0.0:1790701353.973070:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1790701353.973074:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:0.0:1790701353.973078:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1790701353.973081:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:0.0:1790701353.973084:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [67:67:255:4294967295] 192.168.206.116@tcp LPNI seq info [67:67:7:4294967295]
00000400:00000200:0.0:1790701353.973089:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701353.973098:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1790701353.973106:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:0.0:1790701353.973127:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bdb3200.
00000800:00000200:0.0:1790701353.973135:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:0.0:1790701353.973142:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:0.0:1790701353.973146:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bdb3200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1790701353.973168:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701353.973421:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775ba18
00000400:00000010:1.0:1790701353.973426:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775ba18.
00000100:00000200:1.0:1790701353.973438:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910bd3b80 x1877686323519360/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1790701353.974132:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701353.974137:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bdb3200.
00000400:00000200:1.0:1790701353.974143:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701353.974149:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701353.974172:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a440
00000400:00000010:1.0:1790701353.974174:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a440.
00000100:00000200:1.0:1790701353.974183:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910bd0380 x1877686323519232/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701353.979974:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701353.979983:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701353.979992:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701353.980008:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701353.980014:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701353.980017:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea501f80
00000400:00000200:0.0:1790701353.980022:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701353.980028:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x4f1 [1] + 224
00000400:00000200:0.0:1790701353.980057:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701353.980062:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701353.980076:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910bd3b80 x1877686323519360/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701369 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1790701353.980227:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69f20.
00000400:00000200:1.0:1790701353.980236:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bc38
00000400:00000010:1.0:1790701353.980238:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bc38.
00000100:00000200:1.0:1790701353.980245:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910bd3b80 x1877686323519360/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701369 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1790701353.980254:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910bd3b80 x1877686323519360/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701369 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1790701353.980286:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf7800.
02000000:00000010:1.0:1790701353.980292:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c923075a00.
00000100:00000010:1.0:1790701353.980296:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd3b80.
00000400:00000200:1.0:1790701353.980938:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701353.980943:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:1.0:1790701353.980948:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701353.980951:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701353.980954:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701353.980957:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea501f00
00000400:00000200:1.0:1790701353.980961:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701353.980965:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x501 [1] + 224
00000400:00000200:1.0:1790701353.980974:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701353.980977:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:1.0:1790701353.980985:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910bd0380 x1877686323519232/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701369 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1790701353.981072:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2898.
00000400:00000200:0.0:1790701353.981077:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a4c8
00000400:00000010:0.0:1790701353.981079:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a4c8.
00000100:00000200:0.0:1790701353.981085:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910bd0380 x1877686323519232/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701369 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1790701353.981092:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910bd0380 x1877686323519232/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701369 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1790701353.981106:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a04e600.
02000000:00000010:0.0:1790701353.981110:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c923075300.
00000100:00000010:0.0:1790701353.981113:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd0380.
00000100:00000010:0.0:1790701354.925304:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac4e0.
00000100:00000010:1.0:1790701354.925655:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac0c0.
00000001:02000400:0.0:1790701357.809057:0:125480:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 500: Check deadlock on ping target update ========================================================== 13:02:36 (1790701356)
00000001:02000400:0.0:1790701357.886682:0:125485:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000100:00000010:1.0:1790701359.085251:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd3b80.
02000000:00000010:1.0:1790701359.085278:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c923075a00.
00000100:00000010:1.0:1790701359.085318:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd1500.
02000000:00000010:1.0:1790701359.085323:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c923075300.
02000000:00000010:1.0:1790701359.085360:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf6800.
00000400:00000200:1.0:1790701359.085373:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701359.085467:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69f20.
00000400:00000010:1.0:1790701359.085476:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bc38.
00000100:00000200:1.0:1790701359.085495:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323519488, portal 25
00000400:00000010:1.0:1790701359.085501:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775ba18.
00000100:00000200:1.0:1790701359.085504:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323519488, offset 0
00000400:00000200:1.0:1790701359.085515:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701359.085526:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701359.085536:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701359.085543:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701359.085548:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701359.085552:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701359.085555:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701359.085559:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [68:68:256:4294967295] 192.168.206.116@tcp LPNI seq info [68:68:8:4294967295]
00000400:00000200:1.0:1790701359.085563:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701359.085573:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701359.085579:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701359.085584:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075700.
00000800:00000200:1.0:1790701359.085592:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701359.085600:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701359.085605:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075700 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701359.085621:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:1.0:1790701359.085627:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac600.
00000100:00000010:1.0:1790701359.085641:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac0c0.
02000000:00000010:1.0:1790701359.085665:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf7800.
00000400:00000200:1.0:1790701359.085669:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701359.085672:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69898.
00000400:00000010:1.0:1790701359.085676:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775baa0.
00000100:00000200:1.0:1790701359.085680:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323519616, portal 10
00000400:00000010:1.0:1790701359.085686:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775be58.
00000100:00000200:1.0:1790701359.085688:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323519616, offset 0
00000400:00000200:1.0:1790701359.085693:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701359.085697:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701359.085700:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701359.085708:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701359.085711:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701359.085713:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701359.085716:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701359.085718:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [69:69:255:4294967295] 192.168.206.116@tcp LPNI seq info [69:69:7:4294967295]
00000400:00000200:1.0:1790701359.085722:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701359.085728:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701359.085733:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701359.085736:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075e00.
00000800:00000200:1.0:1790701359.085739:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701359.085743:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701359.085746:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075e00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701359.085752:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:1.0:1790701359.085819:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701359.085823:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075700.
00000400:00000200:1.0:1790701359.085831:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701359.085837:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701359.085843:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775ba18
00000400:00000010:1.0:1790701359.085846:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775ba18.
00000100:00000200:1.0:1790701359.085856:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910bd3b80 x1877686323519488/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1790701359.088693:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701359.088706:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075e00.
00000400:00000200:1.0:1790701359.088717:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701359.088728:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701359.088737:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775be58
00000400:00000010:1.0:1790701359.088740:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775be58.
00000100:00000200:1.0:1790701359.088751:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910bd1500 x1877686323519616/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701375 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701359.091314:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701359.091329:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701359.091342:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701359.091348:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701359.091356:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701359.091361:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502080
00000400:00000200:0.0:1790701359.091368:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701359.091411:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x521 [1] + 224
00000400:00000200:0.0:1790701359.091426:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701359.091431:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701359.091460:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910bd1500 x1877686323519616/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701375 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701359.091509:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701359.091513:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701359.091517:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701359.091520:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701359.091523:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701359.091526:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502000
00000400:00000200:0.0:1790701359.091530:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701359.091535:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x511 [1] + 224
00000400:00000200:0.0:1790701359.091545:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701359.091549:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701359.091557:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910bd3b80 x1877686323519488/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701375 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1790701359.091614:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69898.
00000400:00000200:0.0:1790701359.091619:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775baa0
00000400:00000010:0.0:1790701359.091622:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775baa0.
00000100:00000200:0.0:1790701359.091629:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910bd1500 x1877686323519616/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701375 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1790701359.091638:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910bd1500 x1877686323519616/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701375 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1790701359.091673:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69f20.
00000400:00000200:0.0:1790701359.091677:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bc38
00000400:00000010:0.0:1790701359.091679:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bc38.
00000100:00000200:0.0:1790701359.091683:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910bd3b80 x1877686323519488/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701375 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1790701359.091703:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910bd3b80 x1877686323519488/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701375 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1790701359.091721:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf7800.
02000000:00000010:0.0:1790701359.091730:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c923075300.
00000100:00000010:0.0:1790701359.091736:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd1500.
02000000:00000010:0.0:1790701359.091741:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf6800.
02000000:00000010:0.0:1790701359.091744:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c923075a00.
00000100:00000010:0.0:1790701359.091746:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd3b80.
00000001:02000400:3.0F:1790701359.286888:0:125620:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_500: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
00000100:00000010:0.0:1790701360.048486:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac600.
00000100:00000010:1.0:1790701360.048611:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac0c0.
Debug log: 321 lines, 321 kept, 0 dropped, 0 bad.