00000100:00000010:3.0F:1790701564.823347:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074ca80. 02000000:00000010:3.0:1790701564.823385:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441900. 00000100:00000010:3.0:1790701564.823436:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074d880. 02000000:00000010:3.0:1790701564.823442:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441100. 00000100:00000010:3.0:1790701564.823450:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074c700. 02000000:00000010:3.0:1790701564.823455:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441c00. 00000100:00000010:3.0:1790701564.823467:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074f800. 02000000:00000010:3.0:1790701564.823472:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441500. 00000100:02000000:3.0:1790701564.823478:0:121163:0:(import.c:2148:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff8f8cc7fce800: disconnect after 23s idle 02000000:00000010:3.0:1790701564.856540:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1056c00. 00000400:00000200:3.0:1790701564.856559:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701564.856565:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223f78. 00000400:00000010:3.0:1790701564.856576:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0605d8. 00000100:00000200:3.0:1790701564.856587:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537428096, portal 25 00000400:00000010:3.0:1790701564.856595:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060220. 00000100:00000200:3.0:1790701564.856599:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537428096, offset 0 02000000:00000010:2.0F:1790701564.883037:0:123343:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd10cec00. 00000400:00000200:2.0:1790701564.883078:0:123343:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701564.883085:0:123343:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2b00. 00000400:00000200:3.0:1790701564.884778:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701564.884796:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.884809:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701564.884815:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701564.884884:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701564.884890:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701564.884893:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701564.884897:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [62:62:256:4294967295] 192.168.204.157@tcp LPNI seq info [62:62:8:4294967295] 00000400:00000200:3.0:1790701564.884903:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.884914:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701564.884921:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701564.884929:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441700. 00000800:00000200:3.0:1790701564.884937:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701564.884946:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701564.884951:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701564.884978:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701564.884986:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934d3c0. 00000100:00000010:3.0:1790701564.884999:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934d4e0. 02000000:00000010:3.0:1790701564.885022:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1057400. 00000400:00000200:3.0:1790701564.885028:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701564.885032:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223e18. 00000400:00000010:2.0:1790701564.890163:0:123343:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6ad48. 00000100:00000200:2.0:1790701564.890184:0:123343:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537428480, portal 4 00000400:00000010:2.0:1790701564.890196:0:123343:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6add0. 00000100:00000200:2.0:1790701564.890200:0:123343:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1877686537428480, offset 0 00000400:00000200:2.0:1790701564.890213:0:123343:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701564.890227:0:123343:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701564.890238:0:123343:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701564.890244:0:123343:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701564.890250:0:123343:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701564.890268:0:123343:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701564.890271:0:123343:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701564.890276:0:123343:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [63:63:255:4294967295] 192.168.204.157@tcp LPNI seq info [63:63:7:4294967295] 00000400:00000200:2.0:1790701564.890281:0:123343:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701564.890292:0:123343:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701564.890300:0:123343:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701564.890308:0:123343:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24fb00. 00000800:00000200:2.0:1790701564.890316:0:123343:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701564.890327:0:123343:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701564.890348:0:123343:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701564.890358:0:123343:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:2.0:1790701564.890424:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd10cf600. 00000400:00000200:2.0:1790701564.890429:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701564.890432:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2420. 00000400:00000010:2.0:1790701564.890436:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6acc0. 00000100:00000200:2.0:1790701564.890439:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537428224, portal 10 00000400:00000010:2.0:1790701564.890443:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a440. 00000100:00000200:2.0:1790701564.890446:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537428224, offset 0 00000400:00000200:2.0:1790701564.890450:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701564.890454:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701564.890475:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701564.890478:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701564.890480:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701564.890483:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701564.890485:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701564.890488:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [64:64:254:4294967295] 192.168.204.157@tcp LPNI seq info [64:64:6:4294967295] 00000400:00000200:2.0:1790701564.890492:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701564.890498:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701564.890503:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701564.890506:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24fd00. 00000800:00000200:2.0:1790701564.890510:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701564.890514:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701564.890517:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24fd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701564.890529:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:2.0:1790701564.890805:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca672d80. 02000000:00000010:2.0:1790701564.891355:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24f000. 00000100:02000000:2.0:1790701564.891367:0:121163:0:(import.c:2148:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff8f8cc7fce800: disconnect after 23s idle 02000000:00000010:2.0:1790701564.891396:0:123343:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd10ce400. 00000400:00000200:2.0:1790701564.891401:0:123343:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701564.891404:0:123343:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2a50. 00000400:00000010:2.0:1790701564.891410:0:123343:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6abb0. 00000100:00000200:2.0:1790701564.891413:0:123343:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537428608, portal 4 00000400:00000010:2.0:1790701564.891418:0:123343:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a6e8. 00000100:00000200:2.0:1790701564.891421:0:123343:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1877686537428608, offset 0 00000400:00000200:2.0:1790701564.891427:0:123343:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701564.891431:0:123343:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701564.891436:0:123343:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701564.891439:0:123343:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701564.891441:0:123343:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701564.891443:0:123343:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701564.891445:0:123343:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701564.891454:0:123343:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [65:65:253:4294967295] 192.168.204.157@tcp LPNI seq info [65:65:5:4294967295] 00000400:00000200:2.0:1790701564.891459:0:123343:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701564.891465:0:123343:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701564.891470:0:123343:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701564.891474:0:123343:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24f700. 00000800:00000200:2.0:1790701564.891478:0:123343:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701564.891483:0:123343:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701564.891487:0:123343:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24f700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701564.891492:0:123343:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701564.901572:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701564.901586:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441700. 00000400:00000200:3.0:1790701564.901594:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701564.901606:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701564.903016:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060220 00000400:00000010:3.0:1790701564.903023:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060220. 00000100:00000200:3.0:1790701564.903041:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074ca80 x1877686537428096/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701564.903077:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701564.903079:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24fb00. 00000400:00000200:3.0:1790701564.903083:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701564.903087:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701564.903090:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6add0 00000400:00000010:3.0:1790701564.903092:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6add0. 00000100:00000200:3.0:1790701564.903101:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074f800 x1877686537428480/t0(0) o9->lustre-OST0000-osc-ffff8f8cc7fce800@192.168.204.157@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701564.903117:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701564.903120:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24fd00. 00000400:00000200:3.0:1790701564.903123:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701564.903126:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701564.903128:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a440 00000400:00000010:3.0:1790701564.903130:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a440. 00000100:00000200:3.0:1790701564.903136:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074d880 x1877686537428224/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701564.903182:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701564.903191:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24f700. 00000400:00000200:3.0:1790701564.903195:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701564.903199:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701564.903203:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a6e8 00000400:00000010:3.0:1790701564.903205:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a6e8. 00000100:00000200:3.0:1790701564.903212:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca672d80 x1877686537428608/t0(0) o9->lustre-OST0001-osc-ffff8f8cc7fce800@192.168.204.157@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701564.903259:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060220. 00000100:00000200:3.0:1790701564.903272:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537428352, portal 10 00000400:00000010:3.0:1790701564.903565:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060088. 00000100:00000200:3.0:1790701564.903580:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537428352, offset 0 00000400:00000200:3.0:1790701564.903594:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701564.903604:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.903613:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701564.903618:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701564.903622:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701564.903625:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701564.903628:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701564.903632:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [66:66:256:4294967295] 192.168.204.157@tcp LPNI seq info [66:66:8:4294967295] 00000400:00000200:3.0:1790701564.903638:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.903648:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701564.903654:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701564.903660:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441700. 00000800:00000200:3.0:1790701564.903669:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701564.903678:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701564.903693:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701564.903726:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701564.904044:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701564.904048:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441700. 00000400:00000200:3.0:1790701564.904053:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701564.904059:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701564.904075:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060088 00000400:00000010:3.0:1790701564.904078:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060088. 00000100:00000200:3.0:1790701564.904088:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074c700 x1877686537428352/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701564.912193:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912207:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701564.912218:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912225:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912232:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912659:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7101d00 00000400:00000200:3.0:1790701564.912677:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912701:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x495 [1] + 224 00000400:00000200:3.0:1790701564.912727:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701564.912734:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701564.912762:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074d880 x1877686537428224/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701580 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701564.912807:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912812:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701564.912856:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912860:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912864:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912867:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7101d80 00000400:00000200:3.0:1790701564.912871:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912886:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x4b5 [1] + 224 00000400:00000200:3.0:1790701564.912893:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701564.912897:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701564.912906:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074c700 x1877686537428352/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701580 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701564.912934:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912954:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701564.912960:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912964:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912969:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912972:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 4 MB=0x6abbef7101e00 00000400:00000200:3.0:1790701564.912977:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.912982:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.157@tcp of length 224/224 into md 0x485 [1] + 224 00000400:00000200:3.0:1790701564.912991:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701564.912995:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701564.913002:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074f800 x1877686537428480/t0(0) o9->lustre-OST0000-osc-ffff8f8cc7fce800@192.168.204.157@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701574 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701564.913025:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.913028:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701564.913032:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.913035:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.913038:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.913041:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 4 MB=0x6abbef7101e80 00000400:00000200:3.0:1790701564.913045:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.913048:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.157@tcp of length 224/224 into md 0x4a5 [1] + 224 00000400:00000200:3.0:1790701564.913058:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701564.913062:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701564.913069:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca672d80 x1877686537428608/t0(0) o9->lustre-OST0001-osc-ffff8f8cc7fce800@192.168.204.157@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701574 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701564.913105:0:123343:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2a50. 00000400:00000200:3.0:1790701564.913110:0:123343:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6abb0 00000400:00000010:3.0:1790701564.913112:0:123343:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6abb0. 00000100:00000200:3.0:1790701564.913119:0:123343:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca672d80 x1877686537428608/t0(0) o9->lustre-OST0001-osc-ffff8f8cc7fce800@192.168.204.157@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701574 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701564.913128:0:123343:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca672d80 x1877686537428608/t0(0) o9->lustre-OST0001-osc-ffff8f8cc7fce800@192.168.204.157@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701574 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701564.913168:0:123343:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2b00. 00000400:00000200:3.0:1790701564.913172:0:123343:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6ad48 00000400:00000010:3.0:1790701564.913174:0:123343:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6ad48. 00000100:00000200:3.0:1790701564.913179:0:123343:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074f800 x1877686537428480/t0(0) o9->lustre-OST0000-osc-ffff8f8cc7fce800@192.168.204.157@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701574 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701564.913186:0:123343:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074f800 x1877686537428480/t0(0) o9->lustre-OST0000-osc-ffff8f8cc7fce800@192.168.204.157@tcp:28/4 lens 224/224 e 0 to 0 dl 1790701574 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701564.913203:0:123343:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd10ce400. 02000000:00000010:3.0:1790701564.913209:0:123343:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24f000. 00000100:00000010:3.0:1790701564.913214:0:123343:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca672d80. 02000000:00000010:3.0:1790701564.913219:0:123343:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd10cec00. 02000000:00000010:3.0:1790701564.913222:0:123343:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441500. 00000100:00000010:3.0:1790701564.913224:0:123343:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074f800. 00000400:00000010:3.0:1790701564.913243:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223e18. 00000400:00000200:3.0:1790701564.913246:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060220 00000400:00000010:3.0:1790701564.913248:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060220. 00000100:00000200:3.0:1790701564.913253:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074c700 x1877686537428352/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701580 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701564.913262:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074c700 x1877686537428352/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701580 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701564.913274:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1057400. 02000000:00000010:3.0:1790701564.913278:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441c00. 00000100:00000010:3.0:1790701564.913281:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074c700. 00000400:00000010:3.0:1790701564.915272:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2420. 00000400:00000200:3.0:1790701564.915281:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6acc0 00000400:00000010:3.0:1790701564.915390:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6acc0. 00000100:00000200:3.0:1790701564.915467:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074d880 x1877686537428224/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701580 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701564.915477:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074d880 x1877686537428224/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701580 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701564.915518:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd10cf600. 02000000:00000010:3.0:1790701564.915523:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441100. 00000100:00000010:3.0:1790701564.915527:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074d880. 00000400:00000200:3.0:1790701564.916594:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.916603:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701564.916612:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.916617:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.916622:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.916627:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7101c80 00000400:00000200:3.0:1790701564.916632:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701564.916638:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x475 [1] + 224 00000400:00000200:3.0:1790701564.916656:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701564.916661:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701564.916673:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074ca80 x1877686537428096/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701580 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701564.916734:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223f78. 00000400:00000200:3.0:1790701564.916739:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0605d8 00000400:00000010:3.0:1790701564.916760:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0605d8. 00000100:00000200:3.0:1790701564.916768:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074ca80 x1877686537428096/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701580 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701564.916777:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074ca80 x1877686537428096/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701580 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701564.916799:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1056c00. 02000000:00000010:3.0:1790701564.916803:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441900. 00000100:00000010:3.0:1790701564.916807:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074ca80. 00000100:00000010:2.0:1790701566.221533:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc934d3c0. 00000100:00000010:3.0:1790701566.221872:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc934d4e0. 00000001:02000400:0.0F:1790701569.630518:0:126334:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 475: per-operation latency stats populate and reset ========================================================== 13:06:07 (1790701567) 00000001:02000400:1.0F:1790701569.791761:0:126348:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure 00000100:00000010:0.0:1790701569.938124:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e2300. 02000000:00000010:0.0:1790701569.938404:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe280a00. 00000100:00000010:0.0:1790701569.938443:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e1c00. 02000000:00000010:0.0:1790701569.938449:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe280d00. 00000100:00000010:0.0:1790701569.938456:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e1180. 02000000:00000010:0.0:1790701569.938459:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe280400. 02000000:00000010:0.0:1790701569.938496:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc83bc200. 00000400:00000200:0.0:1790701569.938507:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701569.938511:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360fa50. 00000400:00000010:0.0:1790701569.938519:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb3b8. 00000100:00000200:0.0:1790701569.938528:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537428736, portal 25 00000400:00000010:0.0:1790701569.938534:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb220. 00000100:00000200:0.0:1790701569.938537:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537428736, offset 0 00000400:00000200:0.0:1790701569.938547:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701569.938556:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701569.938565:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701569.938570:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701569.938575:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701569.938579:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701569.938582:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701569.940119:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [67:67:256:4294967295] 192.168.204.157@tcp LPNI seq info [67:67:8:4294967295] 00000400:00000200:0.0:1790701569.940128:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701569.940140:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701569.940147:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701569.940156:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe280200. 00000800:00000200:0.0:1790701569.940163:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701569.940172:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701569.940176:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe280200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701569.940191:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:0.0:1790701569.940196:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ccb863ae0. 00000100:00000010:0.0:1790701569.940233:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ccb863720. 02000000:00000010:0.0:1790701569.940253:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc83bcc00. 00000400:00000200:0.0:1790701569.940258:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701569.940262:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f630. 00000400:00000010:0.0:1790701569.940267:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbaa0. 00000100:00000200:0.0:1790701569.940271:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537428992, portal 10 00000400:00000010:0.0:1790701569.940278:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb7f8. 00000100:00000200:0.0:1790701569.940281:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537428992, offset 0 00000400:00000200:0.0:1790701569.940287:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701569.940292:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701569.940299:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701569.940303:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701569.940307:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701569.940309:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701569.940312:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701569.940314:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [68:68:255:4294967295] 192.168.204.157@tcp LPNI seq info [68:68:7:4294967295] 00000400:00000200:0.0:1790701569.940317:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701569.940323:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701569.940327:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701569.940330:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe280600. 00000800:00000200:0.0:1790701569.940333:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701569.940337:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701569.940340:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe280600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701569.940346:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:0.0:1790701569.940370:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc83bd400. 00000400:00000200:0.0:1790701569.940374:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701569.940376:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f420. 00000400:00000010:0.0:1790701569.940379:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbee0. 00000100:00000200:0.0:1790701569.940382:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537428864, portal 10 00000400:00000010:0.0:1790701569.940386:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecba18. 00000100:00000200:0.0:1790701569.940388:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537428864, offset 0 00000400:00000200:0.0:1790701569.940400:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701569.940404:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701569.940407:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701569.940409:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701569.940411:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701569.940414:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701569.940416:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701569.940419:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [69:69:254:4294967295] 192.168.204.157@tcp LPNI seq info [69:69:6:4294967295] 00000400:00000200:0.0:1790701569.940422:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701569.940428:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701569.940432:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701569.940437:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe280300. 00000800:00000200:0.0:1790701569.940441:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701569.940444:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701569.940447:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe280300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701569.940451:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701569.940784:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701569.940790:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe280200. 00000400:00000200:3.0:1790701569.940798:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701569.940812:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701569.940848:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb220 00000400:00000010:3.0:1790701569.940851:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb220. 00000100:00000200:3.0:1790701569.940863:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e2300 x1877686537428736/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701585 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701569.940887:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701569.940889:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe280600. 00000400:00000200:3.0:1790701569.940892:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701569.940895:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701569.940897:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb7f8 00000400:00000010:3.0:1790701569.940898:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb7f8. 00000100:00000200:3.0:1790701569.940902:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e1180 x1877686537428992/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701585 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701569.940969:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701569.940971:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe280300. 00000400:00000200:3.0:1790701569.940974:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701569.940977:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701569.940979:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecba18 00000400:00000010:3.0:1790701569.940982:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecba18. 00000100:00000200:3.0:1790701569.940998:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e1c00 x1877686537428864/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701585 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701569.952835:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.952847:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701569.952855:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.952859:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.952866:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.952872:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102000 00000400:00000200:3.0:1790701569.952877:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.952883:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x4cd [1] + 224 00000400:00000200:3.0:1790701569.952895:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701569.952899:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701569.956571:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e1180 x1877686537428992/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701585 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701569.956625:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.956631:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701569.956637:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.956640:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.956655:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.956660:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7101f80 00000400:00000200:3.0:1790701569.956665:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.956672:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x4d5 [1] + 224 00000400:00000200:3.0:1790701569.956683:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701569.956688:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701569.956694:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e1c00 x1877686537428864/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701585 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701569.956708:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.956712:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701569.956716:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.956718:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.956720:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.956723:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7101f00 00000400:00000200:3.0:1790701569.956725:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701569.956730:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x4c5 [1] + 224 00000400:00000200:3.0:1790701569.956739:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701569.956742:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701569.956748:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e2300 x1877686537428736/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701585 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701569.958377:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f420. 00000400:00000200:1.0:1790701569.958390:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbee0 00000400:00000010:1.0:1790701569.958393:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbee0. 00000100:00000200:1.0:1790701569.958407:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e1c00 x1877686537428864/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701585 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701569.958418:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e1c00 x1877686537428864/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701585 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701569.958452:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360fa50. 00000400:00000200:1.0:1790701569.958455:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb3b8 00000400:00000010:1.0:1790701569.958457:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb3b8. 00000100:00000200:1.0:1790701569.958462:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e2300 x1877686537428736/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701585 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701569.958471:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e2300 x1877686537428736/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701585 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701569.958481:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f630. 00000400:00000200:1.0:1790701569.958484:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbaa0 00000400:00000010:1.0:1790701569.958486:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbaa0. 00000100:00000200:1.0:1790701569.958490:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e1180 x1877686537428992/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701585 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701569.958499:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e1180 x1877686537428992/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701585 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701569.958516:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc83bd400. 02000000:00000010:1.0:1790701569.958522:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe280d00. 00000100:00000010:1.0:1790701569.958527:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e1c00. 02000000:00000010:1.0:1790701569.958531:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc83bc200. 02000000:00000010:1.0:1790701569.958535:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe280a00. 00000100:00000010:1.0:1790701569.958538:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e2300. 02000000:00000010:1.0:1790701569.958541:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc83bcc00. 02000000:00000010:1.0:1790701569.958543:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe280400. 00000100:00000010:1.0:1790701569.958559:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e1180. 00000100:00000010:2.0:1790701571.276154:0:120878:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ccb863ae0. 00000100:00000010:3.0:1790701571.276762:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ccb863720. 00000001:02000400:3.0:1790701571.345709:0:126476:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_475: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240 Debug log: 421 lines, 421 kept, 0 dropped, 0 bad.