00000100:00000010:0.0F:1790701318.126485:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a22d80. 02000000:00000010:0.0:1790701318.126515:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3400. 00000100:00000010:0.0:1790701318.126548:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a21f80. 02000000:00000010:0.0:1790701318.126552:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3500. 00000100:00000010:0.0:1790701318.126559:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a22300. 02000000:00000010:0.0:1790701318.126562:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3b00. 00000100:00000010:0.0:1790701318.126567:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a23100. 02000000:00000010:0.0:1790701318.126570:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3e00. 02000000:00000010:0.0:1790701318.126595:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c919bb3400. 00000400:00000200:0.0:1790701318.126604:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701318.126608:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2bb0. 00000400:00000010:0.0:1790701318.126614:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701318.126620:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323517056, portal 10 00000400:00000010:0.0:1790701318.126627:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45acc0. 00000100:00000200:0.0:1790701318.126631:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323517056, offset 0 00000400:00000200:0.0:1790701318.126639:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701318.126647:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.126664:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701318.126668:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701318.126672:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701318.126675:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701318.126678:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701318.126682:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [48:48:256:4294967295] 192.168.206.116@tcp LPNI seq info [48:48:8:4294967295] 00000400:00000200:0.0:1790701318.126686:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.126695:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701318.126700:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701318.126705:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bdb3300. 00000800:00000200:0.0:1790701318.126712:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701318.126719:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701318.126723:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bdb3300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701318.126737:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:0.0:1790701318.126742:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c921d10480. 00000100:00000010:0.0:1790701318.126756:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c921d10960. 02000000:00000010:0.0:1790701318.126766:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c919bb2400. 00000400:00000200:0.0:1790701318.126770:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701318.126772:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2f20. 00000400:00000010:0.0:1790701318.126776:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a440. 00000100:00000200:0.0:1790701318.126778:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323517312, portal 4 00000020:00000010:0.0:1790701318.126785:0:121246:0:(lprocfs_status.c:1413:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff98c931de6000. 00000400:00000010:0.0:1790701318.126789:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a7f8. 00000100:00000200:0.0:1790701318.126791:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1877686323517312, offset 0 00000400:00000200:0.0:1790701318.126796:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701318.126799:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.126803:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701318.126806:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701318.126808:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701318.126810:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701318.126812:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701318.126815:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [49:49:255:4294967295] 192.168.206.116@tcp LPNI seq info [49:49:7:4294967295] 00000400:00000200:0.0:1790701318.126819:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.126825:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701318.126829:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701318.126832:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bdb3000. 00000800:00000200:0.0:1790701318.126836:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701318.126840:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701318.126843:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bdb3000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701318.126847:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 02000000:00000010:0.0:1790701318.126861:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c919bb2a00. 00000400:00000200:0.0:1790701318.126864:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701318.126867:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2000. 00000400:00000010:0.0:1790701318.126870:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45ad48. 00000100:00000200:0.0:1790701318.126872:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323516928, portal 25 00000400:00000010:0.0:1790701318.126875:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a660. 00000100:00000200:0.0:1790701318.126882:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323516928, offset 0 00000400:00000200:0.0:1790701318.126886:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701318.126890:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.126893:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701318.126896:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701318.126898:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701318.126900:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701318.126902:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701318.126905:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [50:50:254:4294967295] 192.168.206.116@tcp LPNI seq info [50:50:6:4294967295] 00000400:00000200:0.0:1790701318.126909:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.126915:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701318.126919:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701318.126922:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bdb3c00. 00000800:00000200:0.0:1790701318.126925:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701318.126929:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701318.126932:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bdb3c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701318.126936:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 02000000:00000010:0.0:1790701318.126946:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c919bb2600. 00000400:00000200:0.0:1790701318.126950:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:0.0:1790701318.126952:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c910cb2898. 00000400:00000010:0.0:1790701318.126954:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a4c8. 00000100:00000200:0.0:1790701318.126957:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323517184, portal 4 00000020:00000010:0.0:1790701318.126962:0:121246:0:(lprocfs_status.c:1413:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff98c931de2000. 00000400:00000010:0.0:1790701318.126966:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c92b45a880. 00000100:00000200:0.0:1790701318.126968:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1877686323517184, offset 0 00000400:00000200:0.0:1790701318.126971:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:0.0:1790701318.126974:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.126977:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:0.0:1790701318.126979:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701318.126981:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:0.0:1790701318.126984:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701318.126991:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:0.0:1790701318.126994:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [51:51:253:4294967295] 192.168.206.116@tcp LPNI seq info [51:51:5:4294967295] 00000400:00000200:0.0:1790701318.126998:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.127004:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:0.0:1790701318.127008:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:0.0:1790701318.127011:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c93bdb3f00. 00000800:00000200:0.0:1790701318.127015:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:0.0:1790701318.127018:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:0.0:1790701318.127021:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c93bdb3f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701318.127026:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:0.0:1790701318.127056:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701318.127059:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bdb3300. 00000400:00000200:0.0:1790701318.127063:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701318.127067:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701318.127072:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45acc0 00000400:00000010:0.0:1790701318.127074:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45acc0. 00000100:00000200:0.0:1790701318.127081:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a21f80 x1877686323517056/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701318.127098:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701318.127100:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bdb3000. 00000400:00000200:0.0:1790701318.127102:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701318.127105:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701318.127374:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a7f8 00000400:00000010:0.0:1790701318.127378:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a7f8. 00000100:00000200:0.0:1790701318.127385:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a23100 x1877686323517312/t0(0) o400->lustre-OST0001-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701318.127403:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701318.127406:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bdb3c00. 00000400:00000200:0.0:1790701318.127409:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701318.127412:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701318.127415:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a660 00000400:00000010:0.0:1790701318.127423:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a660. 00000100:00000200:0.0:1790701318.127428:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a22d80 x1877686323516928/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1790701318.127768:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1790701318.127771:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c93bdb3f00. 00000400:00000200:0.0:1790701318.127774:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701318.127779:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:0.0:1790701318.127782:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a880 00000400:00000010:0.0:1790701318.127784:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a880. 00000100:00000200:0.0:1790701318.127789:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a22300 x1877686323517184/t0(0) o400->lustre-OST0000-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701334 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701318.150136:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150146:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701318.150174:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150179:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150185:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150189:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea501680 00000400:00000200:0.0:1790701318.150194:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150200:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x421 [1] + 224 00000400:00000200:0.0:1790701318.150212:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701318.150226:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701318.150244:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a21f80 x1877686323517056/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701318.150281:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2bb0. 00000400:00000200:0.0:1790701318.150284:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a770 00000400:00000010:0.0:1790701318.150287:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a770. 00000100:00000200:0.0:1790701318.150292:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a21f80 x1877686323517056/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701318.150300:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a21f80 x1877686323517056/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701318.150344:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c919bb3400. 02000000:00000010:0.0:1790701318.150349:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3500. 00000100:00000010:0.0:1790701318.150353:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a21f80. 00000400:00000200:0.0:1790701318.150374:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150378:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701318.150383:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150386:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150389:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150392:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 4 MB=0x6abbeea501780 00000400:00000200:0.0:1790701318.150396:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150400:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.206.116@tcp of length 224/224 into md 0x431 [1] + 224 00000400:00000200:0.0:1790701318.150406:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701318.150409:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701318.150416:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a23100 x1877686323517312/t0(0) o400->lustre-OST0001-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701318.150434:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2f20. 00000400:00000200:0.0:1790701318.150437:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a440 00000400:00000010:0.0:1790701318.150438:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a440. 00000100:00000200:0.0:1790701318.150442:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a23100 x1877686323517312/t0(0) o400->lustre-OST0001-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701318.150450:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a23100 x1877686323517312/t0(0) o400->lustre-OST0001-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701318.150462:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c919bb2400. 02000000:00000010:0.0:1790701318.150465:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3e00. 00000100:00000010:0.0:1790701318.150467:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a23100. 00000400:00000200:0.0:1790701318.150479:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150482:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701318.150486:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150495:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150499:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150502:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 4 MB=0x6abbeea501700 00000400:00000200:0.0:1790701318.150505:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.150508:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.206.116@tcp of length 224/224 into md 0x451 [1] + 224 00000400:00000200:0.0:1790701318.150526:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701318.150530:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701318.150536:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a22300 x1877686323517184/t0(0) o400->lustre-OST0000-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701318.150554:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2898. 00000400:00000200:0.0:1790701318.150557:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45a4c8 00000400:00000010:0.0:1790701318.150558:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45a4c8. 00000100:00000200:0.0:1790701318.150562:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a22300 x1877686323517184/t0(0) o400->lustre-OST0000-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701318.150570:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a22300 x1877686323517184/t0(0) o400->lustre-OST0000-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701318.150583:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c919bb2600. 02000000:00000010:0.0:1790701318.150586:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3b00. 00000100:00000010:0.0:1790701318.150595:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a22300. 00000400:00000200:0.0:1790701318.152551:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.152559:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701318.152566:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.152570:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.152575:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.152579:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea501600 00000400:00000200:0.0:1790701318.152594:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701318.152600:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x441 [1] + 224 00000400:00000200:0.0:1790701318.152612:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701318.152615:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701318.152626:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a22d80 x1877686323516928/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0F:1790701318.153001:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c910cb2000. 00000400:00000200:1.0:1790701318.153011:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c92b45ad48 00000400:00000010:1.0:1790701318.153013:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c92b45ad48. 00000100:00000200:1.0:1790701318.153020:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a22d80 x1877686323516928/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701318.153027:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a22d80 x1877686323516928/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701334 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701318.153051:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c919bb2a00. 02000000:00000010:1.0:1790701318.153055:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3400. 00000100:00000010:1.0:1790701318.153059:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a22d80. 00000100:00000010:1.0:1790701319.089799:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c921d10960. 00000100:00000010:0.0:1790701319.101723:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c921d10480. 00000100:00000010:0.0:1790701323.244284:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a22300. 02000000:00000010:0.0:1790701323.244305:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3b00. 00000100:00000010:0.0:1790701323.244339:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a23100. 02000000:00000010:0.0:1790701323.244344:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3e00. 02000000:00000010:1.0:1790701323.244354:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c918df2200. 00000100:00000010:0.0:1790701323.244354:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a21f80. 02000000:00000010:0.0:1790701323.244360:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3500. 00000100:02000000:0.0:1790701323.244365:0:37:0:(import.c:2148:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff98c907b8c000: disconnect after 23s idle 00000400:00000200:1.0:1790701323.244369:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701323.244376:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e693c8. 00000400:00000010:1.0:1790701323.244384:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b110. 00000100:00000200:1.0:1790701323.244392:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323517440, portal 25 00000400:00000010:1.0:1790701323.244397:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775baa0. 00000100:00000200:1.0:1790701323.244401:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323517440, offset 0 00000400:00000200:1.0:1790701323.244409:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701323.244417:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.244425:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701323.244429:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701323.244433:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701323.244437:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701323.244439:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701323.244443:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [52:52:256:4294967295] 192.168.206.116@tcp LPNI seq info [52:52:8:4294967295] 00000400:00000200:1.0:1790701323.244448:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.244456:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701323.244462:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701323.244467:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075e00. 00000800:00000200:1.0:1790701323.244474:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701323.244481:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701323.244497:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701323.244512:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:1.0:1790701323.244517:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac420. 00000100:00000010:1.0:1790701323.244528:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cacf60. 02000000:00000010:1.0:1790701323.244547:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c918df2800. 00000400:00000200:1.0:1790701323.244551:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:1.0:1790701323.244554:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69f20. 00000400:00000010:1.0:1790701323.244559:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bc38. 00000100:00000200:1.0:1790701323.244562:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323517568, portal 10 00000400:00000010:1.0:1790701323.244567:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775ba18. 00000100:00000200:1.0:1790701323.244570:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323517568, offset 0 00000400:00000200:1.0:1790701323.244574:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:1.0:1790701323.244577:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.244581:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:1.0:1790701323.244583:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701323.244586:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:1.0:1790701323.244588:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701323.244590:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:1.0:1790701323.244593:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [53:53:255:4294967295] 192.168.206.116@tcp LPNI seq info [53:53:7:4294967295] 00000400:00000200:1.0:1790701323.244597:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.244602:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:1.0:1790701323.244605:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:1.0:1790701323.244609:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075b00. 00000800:00000200:1.0:1790701323.244613:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:1.0:1790701323.244617:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:1.0:1790701323.244619:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701323.244625:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701323.244792:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701323.244796:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075e00. 00000400:00000200:1.0:1790701323.244802:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701323.244807:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701323.244819:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775baa0 00000400:00000010:1.0:1790701323.244822:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775baa0. 00000100:00000200:1.0:1790701323.244830:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a22300 x1877686323517440/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701323.244920:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701323.244924:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075b00. 00000400:00000200:1.0:1790701323.244927:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701323.244932:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701323.244935:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775ba18 00000400:00000010:1.0:1790701323.244936:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775ba18. 00000100:00000200:1.0:1790701323.244943:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a23100 x1877686323517568/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701323.249858:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.249868:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701323.249875:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.249893:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.249899:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.249902:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea501800 00000400:00000200:1.0:1790701323.249907:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.249912:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x461 [1] + 224 00000400:00000200:1.0:1790701323.249923:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701323.249927:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701323.249939:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a22300 x1877686323517440/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701339 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701323.249965:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.249968:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701323.249973:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.249975:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.249977:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.249980:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea501880 00000400:00000200:1.0:1790701323.249997:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.250001:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x471 [1] + 224 00000400:00000200:1.0:1790701323.250010:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701323.250027:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701323.250034:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a23100 x1877686323517568/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701339 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701323.250062:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69f20. 00000400:00000200:1.0:1790701323.250067:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bc38 00000400:00000010:1.0:1790701323.250070:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bc38. 00000100:00000200:1.0:1790701323.250076:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a23100 x1877686323517568/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701339 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701323.250085:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a23100 x1877686323517568/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701339 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701323.250110:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e693c8. 00000400:00000200:1.0:1790701323.250113:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b110 00000400:00000010:1.0:1790701323.250115:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b110. 00000100:00000200:1.0:1790701323.250120:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a22300 x1877686323517440/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701339 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701323.250128:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a22300 x1877686323517440/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701339 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701323.250141:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c918df2800. 02000000:00000010:1.0:1790701323.250146:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3e00. 00000100:00000010:1.0:1790701323.250150:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a23100. 02000000:00000010:1.0:1790701323.250182:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c918df2200. 02000000:00000010:1.0:1790701323.250185:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3b00. 00000100:00000010:1.0:1790701323.250188:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a22300. 00000100:00000010:0.0:1790701323.264446:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a20e00. 02000000:00000010:0.0:1790701323.264463:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3f00. 00000100:02000000:0.0:1790701323.264470:0:37:0:(import.c:2148:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff98c907b8c000: disconnect after 23s idle 02000000:00000010:2.0F:1790701323.264887:0:121244:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a042800. 00000400:00000200:2.0:1790701323.264910:0:121244:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701323.264913:0:121244:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823f20. 00000400:00000010:2.0:1790701323.264920:0:121244:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeee58. 00000100:00000200:2.0:1790701323.264927:0:121244:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323517696, portal 4 00000400:00000010:2.0:1790701323.264934:0:121244:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee220. 00000100:00000200:2.0:1790701323.264938:0:121244:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1877686323517696, offset 0 00000400:00000200:2.0:1790701323.264946:0:121244:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701323.264954:0:121244:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701323.264961:0:121244:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701323.264966:0:121244:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701323.264970:0:121244:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701323.264973:0:121244:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701323.264976:0:121244:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701323.264980:0:121244:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [54:54:256:4294967295] 192.168.206.116@tcp LPNI seq info [54:54:8:4294967295] 00000400:00000200:2.0:1790701323.264984:0:121244:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701323.264994:0:121244:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701323.265000:0:121244:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701323.265004:0:121244:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1300. 00000800:00000200:2.0:1790701323.265011:0:121244:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701323.265018:0:121244:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701323.265022:0:121244:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701323.265040:0:121244:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 02000000:00000010:2.0:1790701323.265052:0:121244:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a042600. 00000400:00000200:2.0:1790701323.265055:0:121244:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701323.265058:0:121244:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823948. 00000400:00000010:2.0:1790701323.265061:0:121244:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeeee0. 00000100:00000200:2.0:1790701323.265064:0:121244:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323517824, portal 4 00000400:00000010:2.0:1790701323.265068:0:121244:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeeb28. 00000100:00000200:2.0:1790701323.265071:0:121244:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1877686323517824, offset 0 00000400:00000200:2.0:1790701323.265075:0:121244:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701323.265078:0:121244:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701323.265088:0:121244:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701323.265091:0:121244:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701323.265093:0:121244:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701323.265096:0:121244:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701323.265098:0:121244:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701323.265101:0:121244:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [55:55:255:4294967295] 192.168.206.116@tcp LPNI seq info [55:55:7:4294967295] 00000400:00000200:2.0:1790701323.265106:0:121244:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701323.265112:0:121244:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701323.265117:0:121244:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701323.265120:0:121244:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1400. 00000800:00000200:2.0:1790701323.265124:0:121244:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701323.265129:0:121244:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701323.265131:0:121244:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701323.265137:0:121244:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701323.265242:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701323.265246:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1300. 00000400:00000200:1.0:1790701323.265262:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701323.265272:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701323.265277:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee220 00000400:00000010:1.0:1790701323.265279:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee220. 00000100:00000200:1.0:1790701323.265286:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a21f80 x1877686323517696/t0(0) o9->lustre-OST0000-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701323.265342:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701323.265345:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1400. 00000400:00000200:1.0:1790701323.265348:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701323.265351:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701323.265354:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeeb28 00000400:00000010:1.0:1790701323.265356:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeeb28. 00000100:00000200:1.0:1790701323.265360:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c919a20e00 x1877686323517824/t0(0) o9->lustre-OST0001-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701323.269702:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.269806:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701323.269815:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.269820:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.269825:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.269829:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 4 MB=0x6abbeea501900 00000400:00000200:1.0:1790701323.269834:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.269839:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.206.116@tcp of length 224/224 into md 0x481 [1] + 224 00000400:00000200:1.0:1790701323.269867:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701323.269872:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701323.269883:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a21f80 x1877686323517696/t0(0) o9->lustre-OST0000-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701333 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1790701323.271104:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.271109:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:1.0:1790701323.271125:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.271129:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.271132:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.271147:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 4 MB=0x6abbeea501980 00000400:00000200:1.0:1790701323.271151:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:1.0:1790701323.271185:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.206.116@tcp of length 224/224 into md 0x489 [1] + 224 00000400:00000200:1.0:1790701323.271232:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701323.271236:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:1.0:1790701323.271254:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c919a20e00 x1877686323517824/t0(0) o9->lustre-OST0001-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701333 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701323.273400:0:121244:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823f20. 00000400:00000200:2.0:1790701323.273411:0:121244:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeee58 00000400:00000010:2.0:1790701323.273414:0:121244:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeee58. 00000100:00000200:2.0:1790701323.273426:0:121244:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a21f80 x1877686323517696/t0(0) o9->lustre-OST0000-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701333 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701323.273437:0:121244:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a21f80 x1877686323517696/t0(0) o9->lustre-OST0000-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701333 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701323.273544:0:121244:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823948. 00000400:00000200:2.0:1790701323.273548:0:121244:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeeee0 00000400:00000010:2.0:1790701323.273549:0:121244:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeeee0. 00000100:00000200:2.0:1790701323.273555:0:121244:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c919a20e00 x1877686323517824/t0(0) o9->lustre-OST0001-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701333 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701323.273563:0:121244:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c919a20e00 x1877686323517824/t0(0) o9->lustre-OST0001-osc-ffff98c907b8c000@192.168.206.116@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701333 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701323.273581:0:121244:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a042800. 02000000:00000010:2.0:1790701323.273601:0:121244:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3500. 00000100:00000010:2.0:1790701323.273606:0:121244:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a21f80. 02000000:00000010:2.0:1790701323.273611:0:121244:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a042600. 02000000:00000010:2.0:1790701323.273613:0:121244:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3f00. 00000100:00000010:2.0:1790701323.273615:0:121244:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a20e00. 00000100:00000010:1.0:1790701324.205270:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cacf60. 00000100:00000010:0.0:1790701324.205463:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac420. 00000100:00000010:2.0:1790701328.367966:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b96300. 02000000:00000010:2.0:1790701328.367986:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1e00. 00000100:00000010:2.0:1790701328.368013:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94000. 02000000:00000010:2.0:1790701328.368017:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1800. 02000000:00000010:2.0:1790701328.368046:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c920d6d200. 00000400:00000200:2.0:1790701328.368055:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701328.368059:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823948. 00000400:00000010:2.0:1790701328.368066:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeeee0. 00000100:00000200:2.0:1790701328.368073:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323518080, portal 10 00000400:00000010:2.0:1790701328.368080:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeee58. 00000100:00000200:2.0:1790701328.368083:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323518080, offset 0 00000400:00000200:2.0:1790701328.368091:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701328.368098:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701328.368106:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701328.368110:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701328.368114:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701328.368118:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701328.368120:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701328.368124:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [56:56:256:4294967295] 192.168.206.116@tcp LPNI seq info [56:56:8:4294967295] 00000400:00000200:2.0:1790701328.368129:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701328.368139:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701328.368145:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701328.368149:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1d00. 00000800:00000200:2.0:1790701328.368177:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701328.368184:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701328.368189:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701328.368210:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:2.0:1790701328.368214:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c90a1b9360. 00000100:00000010:2.0:1790701328.368226:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c90a1b9cc0. 02000000:00000010:2.0:1790701328.368245:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c920d6ca00. 00000400:00000200:2.0:1790701328.368249:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:2.0:1790701328.368251:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823f20. 00000400:00000010:2.0:1790701328.368255:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee550. 00000100:00000200:2.0:1790701328.368258:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323517952, portal 25 00000400:00000010:2.0:1790701328.368261:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee6e8. 00000100:00000200:2.0:1790701328.368263:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323517952, offset 0 00000400:00000200:2.0:1790701328.368267:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:2.0:1790701328.368271:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701328.368275:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:2.0:1790701328.368278:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701328.368280:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:2.0:1790701328.368283:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701328.368285:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:2.0:1790701328.368287:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [57:57:255:4294967295] 192.168.206.116@tcp LPNI seq info [57:57:7:4294967295] 00000400:00000200:2.0:1790701328.368291:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:2.0:1790701328.368297:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:2.0:1790701328.368302:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:2.0:1790701328.368305:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1c00. 00000800:00000200:2.0:1790701328.368308:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:2.0:1790701328.368313:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:2.0:1790701328.368315:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701328.368320:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701328.393633:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701328.393651:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1d00. 00000400:00000200:1.0:1790701328.393657:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701328.393667:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701328.393674:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeee58 00000400:00000010:1.0:1790701328.393676:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeee58. 00000100:00000200:1.0:1790701328.393688:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b94000 x1877686323518080/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701328.393758:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701328.393762:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1c00. 00000400:00000200:1.0:1790701328.393765:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701328.393769:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701328.393772:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee6e8 00000400:00000010:1.0:1790701328.393774:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee6e8. 00000100:00000200:1.0:1790701328.393779:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c910b96300 x1877686323517952/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701328.406215:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701328.406225:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701328.406234:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701328.406239:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701328.406244:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701328.406249:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea501a80 00000400:00000200:0.0:1790701328.406254:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701328.406260:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x491 [1] + 224 00000400:00000200:0.0:1790701328.406271:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701328.406275:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701328.406289:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b94000 x1877686323518080/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701344 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701328.406314:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701328.406318:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701328.406322:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701328.406325:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701328.406335:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701328.406338:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea501a00 00000400:00000200:0.0:1790701328.406342:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701328.406346:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x499 [1] + 224 00000400:00000200:0.0:1790701328.406400:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701328.406404:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701328.406411:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c910b96300 x1877686323517952/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701344 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701328.407824:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823f20. 00000400:00000200:2.0:1790701328.407833:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee550 00000400:00000010:2.0:1790701328.407836:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee550. 00000100:00000200:2.0:1790701328.407847:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b96300 x1877686323517952/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701344 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701328.407877:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b96300 x1877686323517952/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701344 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701328.407900:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823948. 00000400:00000200:2.0:1790701328.407903:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeeee0 00000400:00000010:2.0:1790701328.407905:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeeee0. 00000100:00000200:2.0:1790701328.407909:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c910b94000 x1877686323518080/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701344 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701328.407916:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c910b94000 x1877686323518080/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701344 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701328.407935:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c920d6ca00. 02000000:00000010:2.0:1790701328.407941:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1e00. 00000100:00000010:2.0:1790701328.407945:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b96300. 02000000:00000010:2.0:1790701328.407948:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c920d6d200. 02000000:00000010:2.0:1790701328.407950:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1800. 00000100:00000010:2.0:1790701328.407952:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94000. 00000100:00000010:0.0:1790701329.327662:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c90a1b9360. 00000100:00000010:1.0:1790701329.328295:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c90a1b9cc0. 00000001:02000400:3.0F:1790701330.045974:0:124053:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 475: per-operation latency stats populate and reset ========================================================== 13:02:08 (1790701328) 00000001:02000400:3.0:1790701330.284252:0:124066:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure 00000001:02000400:2.0:1790701332.310126:0:124194:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_475: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240 00000100:00000010:3.0:1790701333.485212:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908daf100. 02000000:00000010:3.0:1790701333.485228:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d656e00. 00000100:00000010:3.0:1790701333.485260:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908dae300. 02000000:00000010:3.0:1790701333.485265:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d656a00. 02000000:00000010:3.0:1790701333.485293:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf6600. 00000400:00000200:3.0:1790701333.485303:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701333.485306:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318. 00000400:00000010:3.0:1790701333.485313:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701333.485319:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323518336, portal 10 00000400:00000010:3.0:1790701333.485327:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35d48. 00000100:00000200:3.0:1790701333.485330:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323518336, offset 0 00000400:00000200:3.0:1790701333.485338:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701333.485346:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701333.485353:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701333.485358:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701333.485363:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701333.485366:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701333.485369:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701333.485372:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [58:58:256:4294967295] 192.168.206.116@tcp LPNI seq info [58:58:8:4294967295] 00000400:00000200:3.0:1790701333.485377:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701333.485386:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701333.485393:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701333.485397:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d656c00. 00000800:00000200:3.0:1790701333.485403:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701333.485410:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701333.485414:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d656c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701333.485434:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000100:00000010:3.0:1790701333.485439:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c93b3e3a80. 00000100:00000010:3.0:1790701333.485449:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c93b3e3660. 02000000:00000010:3.0:1790701333.485475:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf7e00. 00000400:00000200:3.0:1790701333.485480:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000010:3.0:1790701333.485483:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d90b0. 00000400:00000010:3.0:1790701333.485487:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35660. 00000100:00000200:3.0:1790701333.485490:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323518208, portal 25 00000400:00000010:3.0:1790701333.485494:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d354c8. 00000100:00000200:3.0:1790701333.485496:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323518208, offset 0 00000400:00000200:3.0:1790701333.485501:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp 00000400:00000200:3.0:1790701333.485504:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701333.485508:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.206.116@tcp local destination 00000400:00000200:3.0:1790701333.485510:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701333.485513:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp 00000400:00000200:3.0:1790701333.485516:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701333.485518:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp 00000400:00000200:3.0:1790701333.485521:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [59:59:255:4294967295] 192.168.206.116@tcp LPNI seq info [59:59:7:4294967295] 00000400:00000200:3.0:1790701333.485524:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:3.0:1790701333.485530:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) : PUT try# 0 00000800:00000200:3.0:1790701333.485535:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp 00000800:00000010:3.0:1790701333.485538:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d656900. 00000800:00000200:3.0:1790701333.485541:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5) 00000800:00000200:3.0:1790701333.485545:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988 00000800:00000200:3.0:1790701333.485548:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d656900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701333.485553:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000800:00000200:1.0:1790701333.486051:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701333.486057:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d656c00. 00000400:00000200:1.0:1790701333.486062:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701333.486069:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701333.486074:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35d48 00000400:00000010:1.0:1790701333.486076:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35d48. 00000100:00000200:1.0:1790701333.486090:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908dae300 x1877686323518336/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701333.486194:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1790701333.486197:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d656900. 00000400:00000200:1.0:1790701333.486201:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1790701333.486205:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000400:00000200:1.0:1790701333.486208:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d354c8 00000400:00000010:1.0:1790701333.486210:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d354c8. 00000100:00000200:1.0:1790701333.486215:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff98c908daf100 x1877686323518208/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701333.492864:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701333.492874:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701333.492883:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701333.492890:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701333.492896:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701333.492900:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea501b80 00000400:00000200:0.0:1790701333.492906:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701333.492912:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x4a1 [1] + 224 00000400:00000200:0.0:1790701333.492924:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701333.492929:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701333.492944:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908dae300 x1877686323518336/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701349 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1790701333.492982:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701333.492986:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me 00000400:00000200:0.0:1790701333.492991:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701333.492993:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701333.492996:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701333.492999:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea501b00 00000400:00000200:0.0:1790701333.493003:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0 00000400:00000200:0.0:1790701333.493008:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x4a9 [1] + 224 00000400:00000200:0.0:1790701333.493066:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1790701333.493070:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK 00000100:00000200:0.0:1790701333.493078:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff98c908daf100 x1877686323518208/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701349 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701333.493147:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d90b0. 00000400:00000200:3.0:1790701333.493181:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35660 00000400:00000010:3.0:1790701333.493184:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35660. 00000100:00000200:3.0:1790701333.493192:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908daf100 x1877686323518208/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701349 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701333.493202:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908daf100 x1877686323518208/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701349 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701333.493226:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318. 00000400:00000200:3.0:1790701333.493235:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198 00000400:00000010:3.0:1790701333.493236:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198. 00000100:00000200:3.0:1790701333.493242:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff98c908dae300 x1877686323518336/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701349 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701333.493250:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff98c908dae300 x1877686323518336/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701349 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701333.493274:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf7e00. 02000000:00000010:3.0:1790701333.493280:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d656e00. 00000100:00000010:3.0:1790701333.493285:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908daf100. 02000000:00000010:3.0:1790701333.493289:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf6600. 02000000:00000010:3.0:1790701333.493291:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d656a00. 00000100:00000010:3.0:1790701333.493294:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908dae300. Debug log: 633 lines, 633 kept, 0 dropped, 0 bad.